From 6c2bd06f1219b45a1000006cb8ec6d535f1224aa Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Mon, 14 Sep 2026 08:14:16 -0700 Subject: [PATCH 01/35] [None][feat] Add GLM-5.3-Flash support Add the hybrid KDA/sparse-MLA decoder, vision wrapper, checkpoint loading, and MTP support using shared PyTorch modules. Include deployment guidance and focused regression coverage. Signed-off-by: Ruocheng Jia --- .../kernels/kdaDecode/kdaDecodeLegacy.cu | 1 + cpp/tensorrt_llm/thop/kdaDecodeOp.cpp | 4 +- ...yment-guide-for-glm-5.3-flash-on-trtllm.md | 444 +++ docs/source/deployment-guide/index.rst | 1 + docs/source/media/glm_5_3_flash_fp8_perf.png | Bin 0 -> 145567 bytes docs/source/models/supported-models.md | 2 + .../backends/sparse/glm_kpool/__init__.py | 37 + .../backends/sparse/glm_kpool/backend.py | 891 ++++++ .../sparse/glm_kpool/cache_manager.py | 303 ++ .../backends/sparse/glm_kpool/kernels.py | 427 +++ .../backends/sparse/glm_kpool/params.py | 115 + .../attention/backends/sparse/registry.py | 9 + tensorrt_llm/_torch/model_config.py | 12 +- tensorrt_llm/_torch/models/__init__.py | 2 + tensorrt_llm/_torch/models/_arch_index.py | 5 + .../_torch/models/checkpoints/__init__.py | 3 +- .../checkpoints/hf/glm5_next_weight_mapper.py | 265 ++ .../models/checkpoints/hf/weight_loader.py | 3 +- .../_torch/models/modeling_glm5_next.py | 2587 +++++++++++++++++ .../models/modeling_glm5_next_vision.py | 1261 ++++++++ .../_torch/models/modeling_speculative.py | 3 + .../_torch/modules/kimi_kda/_kda_decode.py | 4 +- .../_torch/modules/kimi_kda/kimi_kda_mixer.py | 108 +- tensorrt_llm/_torch/pyexecutor/_util.py | 129 +- .../_torch/pyexecutor/config_utils.py | 83 +- tensorrt_llm/usage/architecture_allowlist.py | 1 + .../references/acceptance_length.yaml | 6 + .../defs/accuracy/references/gsm8k.yaml | 7 + .../defs/accuracy/references/mmmu.yaml | 7 + .../accuracy/test_disaggregated_serving.py | 92 + .../defs/accuracy/test_glm53_flash.py | 426 +++ .../test_lists/test-db/l0_b200.yml | 1 + .../attention/sparse/glm_kpool/__init__.py | 2 + .../sparse/glm_kpool/test_glm_kpool.py | 151 + .../sparse/glm_kpool/test_kernels.py | 267 ++ .../modeling/test_glm5_next_contracts.py | 331 +++ .../checkpoints/hf/test_weight_loader.py | 12 +- .../modules/kimi_kda/test_kda_decode_op.py | 59 +- .../modules/kimi_kda/test_kda_prefill_op.py | 35 +- .../test_kimi_kda_fused_verify_parity.py | 52 +- 40 files changed, 8055 insertions(+), 93 deletions(-) create mode 100644 docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md create mode 100644 docs/source/media/glm_5_3_flash_fp8_perf.png create mode 100644 tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/__init__.py create mode 100644 tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py create mode 100644 tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py create mode 100644 tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py create mode 100644 tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/params.py create mode 100644 tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py create mode 100644 tensorrt_llm/_torch/models/modeling_glm5_next.py create mode 100644 tensorrt_llm/_torch/models/modeling_glm5_next_vision.py create mode 100644 tests/integration/defs/accuracy/test_glm53_flash.py create mode 100644 tests/unittest/_torch/attention/sparse/glm_kpool/__init__.py create mode 100644 tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py create mode 100644 tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py create mode 100644 tests/unittest/_torch/modeling/test_glm5_next_contracts.py diff --git a/cpp/tensorrt_llm/kernels/kdaDecode/kdaDecodeLegacy.cu b/cpp/tensorrt_llm/kernels/kdaDecode/kdaDecodeLegacy.cu index 1fcec87ce933..ca77fc702e29 100644 --- a/cpp/tensorrt_llm/kernels/kdaDecode/kdaDecodeLegacy.cu +++ b/cpp/tensorrt_llm/kernels/kdaDecode/kdaDecodeLegacy.cu @@ -1560,6 +1560,7 @@ void dispatch_kda_decode_heads(KdaDecodeLaunchParams const& p) case 24: dispatch_kda_decode_layout(p); break; case 32: dispatch_kda_decode_layout(p); break; case 48: dispatch_kda_decode_layout(p); break; + case 64: dispatch_kda_decode_layout(p); break; case 96: dispatch_kda_decode_layout(p); break; default: if constexpr (kCompact) diff --git a/cpp/tensorrt_llm/thop/kdaDecodeOp.cpp b/cpp/tensorrt_llm/thop/kdaDecodeOp.cpp index 20f8063f8c74..80cd6cd37684 100644 --- a/cpp/tensorrt_llm/thop/kdaDecodeOp.cpp +++ b/cpp/tensorrt_llm/thop/kdaDecodeOp.cpp @@ -97,9 +97,9 @@ void validate_kda_decode_fusion_inputs(at::Tensor x_q, at::Tensor x_k, at::Tenso int const HV = static_cast(x_v.size(2)); TORCH_CHECK(B > 0, "KDA decode requires a non-empty batch"); bool const supportedHeads = H == 1 || H == 2 || H == 3 || H == 4 || H == 6 || H == 8 || H == 12 || H == 16 - || H == 24 || H == 32 || H == 48 || H == 96; + || H == 24 || H == 32 || H == 48 || H == 64 || H == 96; TORCH_CHECK( - H == HV && supportedHeads, "KDA decode fusion CUDA supports H == HV in {1,2,3,4,6,8,12,16,24,32,48,96}"); + H == HV && supportedHeads, "KDA decode fusion CUDA supports H == HV in {1,2,3,4,6,8,12,16,24,32,48,64,96}"); TORCH_CHECK(x_k.size(1) == B && x_k.size(2) == H && x_v.size(1) == B, "x_q, x_k, and x_v batch/head dimensions are inconsistent"); TORCH_CHECK(HV % H == 0, "HV must be divisible by H"); diff --git a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md new file mode 100644 index 000000000000..2c1e2c63aeb1 --- /dev/null +++ b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md @@ -0,0 +1,444 @@ +# Deployment Guide for GLM-5.3-Flash on TensorRT LLM - Blackwell Hardware + +## Introduction + +This guide describes how to deploy GLM-5.3-Flash with the TensorRT LLM PyTorch backend and the official FP8 checkpoint on NVIDIA Blackwell GPUs. + +GLM-5.3-Flash is a 320B-parameter mixture-of-experts model with 18B active parameters. It combines KDA linear attention with sparse Multi-Latent Attention and is served through `Glm5NextForConditionalGeneration`. The model supports text, image, and video inputs, as well as Multi-Token Prediction (MTP) for speculative decoding. + +### Validated Features + +The following features have been tested on B200 with four GPUs per worker. Disaggregated serving uses separate context and generation workers: + +* CUDA Graph +* Overlap scheduler +* Chunked prefill +* Multi-Token Prediction (MTP) +* Attention data parallelism (see [`enable_attention_dp`](#enable_attention_dp)) +* Disaggregated serving +* Disaggregated serving with MTP +* KV cache block reuse (with periodic Mamba state snapshots, see [`kv_cache_config`](#kv_cache_config)) +* Image and video inputs (see [Multimodal Inputs](#multimodal-inputs)) + +### Limitations + +* Attention data parallelism cannot currently be combined with MTP. Support for this combination is planned for a future update. Use tensor parallelism when enabling MTP. +* FP8 KV cache is not supported. Leave `kv_cache_config.dtype` at `auto` to use BF16. +* KV cache block reuse requires a periodic Mamba snapshot policy; see [`kv_cache_config`](#kv_cache_config). + +See the [Model-Feature Support Matrix](../models/supported-models.md#model-feature-support-matrix-key-models) for the current support status. + +## Prerequisites + +* GPU: 4x NVIDIA B200 (SM100) for aggregated serving; 8x B200 for disaggregated serving. +* OS: Linux +* Drivers: CUDA Driver 575 or later +* Docker with NVIDIA Container Toolkit installed +* Minimum TensorRT LLM version: 1.3.0rc26 +* Install Transformers commit `49995e1a8c76de9158cf4f4e4995ac7454b8f141` in the GLM deployment environment for model configuration and image/video processing: + + ```bash + pip install "git+https://github.com/huggingface/transformers.git@49995e1a8c76de9158cf4f4e4995ac7454b8f141" + ``` +* Install `fla-core` and `einops` in the container: `pip install fla-core einops`. + +## Models + +* FP8 model: [zai-org/GLM-5.3-Flash](https://huggingface.co/zai-org/GLM-5.3-Flash) + +Download the checkpoint to the model directory mounted into the container: + +```bash +git lfs install +git clone https://huggingface.co/zai-org/GLM-5.3-Flash /models/GLM-5.3-Flash +``` + +## MoE Backend Support Matrix + +There are multiple MoE backends inside TensorRT LLM. Here is the support matrix for GLM-5.3-Flash: + +| Device | Checkpoint | Supported moe_backend | +|--------|-----------|----------------------| +| B200/GB200 | FP8 | TRTLLM, DEEPGEMM | + +On B200, the default `moe_config.backend` (`AUTO`) selects `TRTLLM` for this checkpoint. `DEEPGEMM` is also supported. + +## Deployment Steps + +### Run Docker Container + +Run the Docker container using the TensorRT LLM NVIDIA NGC image. + +```bash +docker run --rm -it \ + --ipc=host \ + --gpus all \ + -p 8000:8000 \ + -v /path/to/your/models:/models \ + --name tensorrt_llm \ + nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26 \ + /bin/bash +``` + +Note: + +* You can mount additional directories using the `-v :` flag, such as mounting the downloaded weight paths. +* The command maps port `8000` from the container to your host so you can access the LLM API endpoint from your host. +* See for all available containers. Containers published in the main branch weekly have an `rcN` suffix, while the monthly release with QA tests has no `rcN` suffix. Use the `rc` release to get the latest model and feature support. + +If you want to use the latest main branch, you can build from source: [https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source.html](https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source.html) + +> **All commands below should be run inside the Docker container.** + +### Recommended Performance Settings + +Use these configurations as starting points for 4x B200 with FP8 weights and a BF16 KV cache. Tune the batch size and token budget for your workload; the [Performance](#performance) section lists the settings used for the measured curve. + +#### B200 FP8 Config + +```bash +cat > /tmp/config.yml < /tmp/config.yml < /tmp/config.yml < bench.sh +concurrency_list="1 4 8 16 32 64 128" +multi_round=5 +isl=1024 +osl=1024 +result_dir=/tmp/glm53_flash_output + +for concurrency in ${concurrency_list}; do + num_prompts=$((concurrency * multi_round)) + python -m tensorrt_llm.serve.scripts.benchmark_serving \ + --model zai-org/GLM-5.3-Flash \ + --backend openai \ + --dataset-name "random" \ + --random-input-len ${isl} \ + --random-output-len ${osl} \ + --random-prefix-len 0 \ + --random-ids \ + --num-prompts ${num_prompts} \ + --max-concurrency ${concurrency} \ + --ignore-eos \ + --tokenize-on-client \ + --seed 0 \ + --percentile-metrics "ttft,tpot,itl,e2el" +done +EOF +chmod +x bench.sh +``` + +To save results to files, add these options to each benchmark command: + +```bash +--save-result \ +--result-dir "${result_dir}" \ +--result-filename "concurrency_${concurrency}.json" +``` + +For more benchmarking options see [benchmark_serving.py](https://github.com/NVIDIA/TensorRT-LLM/blob/main/tensorrt_llm/serve/scripts/benchmark_serving.py). + +Run `bench.sh` to begin a serving benchmark. This will take a long time if you run all the concurrencies. + +```bash +./bench.sh +``` + +Sample TensorRT LLM serving benchmark output. Your results may vary due to ongoing software optimizations. + +``` +============ Serving Benchmark Result ============ +Successful requests: 40 +Benchmark duration (s): [result] +Total input tokens: 40960 +Total generated tokens: 40960 +Request throughput (req/s): [result] +Output token throughput (tok/s): [result] +Total Token throughput (tok/s): [result] +User throughput (tok/s): [result] +---------------Time to First Token---------------- +Mean TTFT (ms): [result] +Median TTFT (ms): [result] +P99 TTFT (ms): [result] +-----Time per Output Token (excl. 1st token)------ +Mean TPOT (ms): [result] +Median TPOT (ms): [result] +P99 TPOT (ms): [result] +---------------Inter-token Latency---------------- +Mean ITL (ms): [result] +Median ITL (ms): [result] +P99 ITL (ms): [result] +----------------End-to-end Latency---------------- +Mean E2EL (ms): [result] +Median E2EL (ms): [result] +P99 E2EL (ms): [result] +================================================== +``` + +### Key Metrics + +#### Time to First Token (TTFT) + +The typical time elapsed from when a request is sent until the first output token is generated. + +#### Time Per Output Token (TPOT) and Inter-Token Latency (ITL) + +* TPOT is the typical time required to generate each token *after* the first one. +* ITL is the typical time delay between the completion of one token and the completion of the next. +* Both TPOT and ITL ignore TTFT. With MTP, tokens can arrive in groups, so streaming ITL depends on how tokens are delivered; use TPOT for per-token decode comparisons. + +For a single request, ITLs are the time intervals between tokens, while TPOT is the average of those intervals: + +$$ +\text{TPOT (1 request)} = \text{Avg(ITL)} = \frac{\text{E2E latency} - \text{TTFT}}{\text{Num Output Tokens} - 1} +$$ + +Across requests, **average TPOT** weights each request equally, while **average ITL** averages the recorded inter-token intervals: + +$$ +\text{Avg TPOT (N requests)} = \frac{\text{TPOT}_1 + \text{TPOT}_2 + \cdots + \text{TPOT}_N}{N} +$$ + +$$ +\text{Avg ITL (N requests)} = \frac{\text{Sum of all ITLs across requests}}{\text{Num measured inter-token intervals}} +$$ + +#### End-to-End (E2E) Latency + +The typical total time from when a request is submitted until the final token of the response is received. + +#### Total Token Throughput + +The combined rate at which the system processes both input (prompt) tokens and output (generated) tokens. + +$$ +\text{Total TPS} = \frac{\text{Num Input Tokens}+\text{Num Output Tokens}}{T_{last} - T_{first}} +$$ + +#### Tokens Per Second (TPS) or Output Token Throughput + +How many output tokens the system generates each second. + +$$ +\text{TPS} = \frac{\text{Num Output Tokens}}{T_{last} - T_{first}} +$$ + +## Performance + +The chart shows output-token throughput per GPU versus per-user output speed on 4x B200 with FP8 weights and a BF16 KV cache. Measurements use the `benchmark_serving` client above, ISL 1024 / OSL 1024, random token IDs, seed 0, and greedy decoding. Each point is one run of `5 * concurrency` requests; labels show concurrency. + +To reproduce the curve, use the serving configurations above with `--max_batch_size 128`, `--max_seq_len 8192`, and `cuda_graph_config.max_batch_size: 128`. Keep CUDA graph padding, chunked prefill, and the overlap scheduler enabled; use a cache memory fraction of 0.5 and disable block reuse. Set `--max_num_tokens 16384` for TP4 / EP4 and MTP3, or `4096` for attention DP4 / EP4. + +The horizontal axis is `1000 / mean_tpot_ms`, excluding TTFT. The vertical axis is aggregate output-token throughput divided by four GPUs, excluding input-token throughput. + +![GLM-5.3-Flash FP8 performance on 4x B200](../media/glm_5_3_flash_fp8_perf.png) + +MTP3 improves single-user decode speed from 154.7 to 397.6 tok/s/user. At concurrency 128, all three configurations deliver approximately 5.9K–6.0K output tok/s in aggregate. MTP uses natural acceptance; random-token workloads can have different acceptance rates from real conversations. This sweep covers the plotted concurrency range, not peak throughput or maximum-context validation. diff --git a/docs/source/deployment-guide/index.rst b/docs/source/deployment-guide/index.rst index 29b7458b9bce..b79a7e5d1c0f 100644 --- a/docs/source/deployment-guide/index.rst +++ b/docs/source/deployment-guide/index.rst @@ -37,4 +37,5 @@ The deployment guides below provide more detailed instructions for serving speci deployment-guide-for-qwen3.8-flash-next-on-trtllm.md deployment-guide-for-kimi-k3-on-trtllm.md deployment-guide-for-glm-5-on-trtllm.md + deployment-guide-for-glm-5.3-flash-on-trtllm.md deployment-guide-for-minimax-m3-on-trtllm.md diff --git a/docs/source/media/glm_5_3_flash_fp8_perf.png b/docs/source/media/glm_5_3_flash_fp8_perf.png new file mode 100644 index 0000000000000000000000000000000000000000..92f3571242e3341c47092537d12db4de98b36462 GIT binary patch literal 145567 zcmeFZcRZK<|2M3mK_!H;DxvIAW+-K)RJQEVkS&s}L>VEKG^{=%dz3x1BUusIBT zJ?`hxd47Mt-{bo0zW=%I>yPX6c$_&=pU-i;$LsZ6@5BG%dBvR+j1**KWIN9&$*GZ% zZP6ej+q`Gn7W{;&c6JqCBpu~-9Mx^FI^Ht5afM99#L>>m*3s&^=>exJHyp0p+8pH< z66BZQJz(zWXy+g)AYlDpe}mul#x(&>h0mApF5B&tE;*2qF{Km#Z7^gEWhUD|Ms`L{ zR>L{sSI;dS_4f6xleaqbPUalwCu7hIO#X7`-S6U5x129+Uo=yboy1FSW?NYIbQR1e zc(>`&QO}ho>>{t30g;)B~a?{+VZI^1a1AU)E3p`0UY{;`7$O};+=R_g!p zN_?%|xncMJ_(Ad5vx^7W{`;kNcRSsGf5*lq@_&C#49EYTVd1`b@+28Mjeo!DQ@^VH zyLVsgH2FJKs{6`)DDZ#H$Jqr1cb{09+$4GH_YMI8fsU_V%bJ>YfB*j7?#7K%758=u znKvCs(aGFcR#rydAmYAY7umBp`h-`>s6LJV=Cri5hstplBhO?!3hE@Jq@*b5!uM*X z>-7wU8+kU@F2<5b84e@o&pj6=AMeV&9}+^vt(ml$mX`L}vu81;RRIhAcd44vjZUN( zm9lQ%zFktn){}g-s8xDpWyO(|Y6B}PtMA%#tKOnZf_>jVy%b@zG&S9*sHk|%x^vIt z!xyTDKZc70ONfi_eRuWKsbsCxi-~F+&tAT)c>kVEOG|5VcDC|xw7d6y&xH*aqGj3! zj{jMEChe9}7pzXEGVd`s2qQF zP+NXUDI04~&kla<=jYdZ@CLC$xNqL4+W%C{qE%_v$z;X0B+X&3wZ%815ttNO7WKU| z-#_sP1{b@{(~HYUOB>i&S|;BpDlBw+HAoZt^JcPJieYhqMuYA3>s&^eroQJJA{BcY zlVcQd)#Nja%gbZGbh-IVrcjVc$S;V z$DMw4sPgg%)!lvaWDha;>&t!X%E$M(UeM38+wkdy$Y~`dzkVGX8-7WrUw)gm?lPA$ z!Z)6)f3zPkiYSMhJk^tskk~6^>Z8E3T=#Br(#~f` zRxr$`bbXBp<98<{BxKay!1|5mUIhgO?xDKIvNG9`5m7nyvUZ}ShqdkRUj~+iJuj%gYHny<|t_j)C!*TiB4T+M4LCRz1 zO=LS6j~$s5p(V;%d{+1eqAL}rD+Dpd-8lQOA@cag#^@6QjSUT2j98I}NZ zr%_oxZuoHcLKLs0*BP5d@{=-7zjT^2z1CNAJ2R{Dhr{_tzElR#@q9b6zVdDb!#o`k z!5n9Xr89jQ*XN-cd7i^CKsmCu2w#w6vbO&;9yJnpvIxDB$a!7#~(q#718)&#ShMrk%SH&1Y^M`4zck$EP3 zg<@@O*|xjbjR9>z!2e-eXQtWKM~@zD9#7Uz=U+5dZvIyLSm@?Yjn;Id{ad!wQI_xA zv7;&5Qk&PHK*P~WUtj->k50;|Q>QY_YIeE*Y120U9KZ8tYvNtyFz&L});*k@oOX70 zC+}|ADd;+D9wBbOF+V@wV&p+!V8!~{{0XAsV%#z^GH&0wvjNNeVywc>EUT|H(H?XA z))t1Snyph>^F5YDaToWKG?UBmmN_{&=FKE>;{ zG(+K*E))t@Y1boBO#Ai?6i$@T8mNRldZdS z^vRPa&)i#TP|ap2ef(H`yCTU=+HI~9eNlroBq&JU%8D1gD#N5=OKY+=6ZT*@82J?u;47^I=hY7@+9d)hfWXiJ*4N;^&zLEKhpN@Lw$W^RUm`? z)Sx`Gl=E(ET#Z7_&cZZRDR_~v`})7%AEPS`uaU8S6fldwyBm(Elip1 z*e5KPm8PMlMuP>un})_4|IXw~$|ap#o5+?=mMRev4qHy&+aY`I+yk{`)XB-w=0jF} zB`lah`C`{U0n=(vR%Sn6P!-c!?sAwq{8jq#fvbEJ^`SdoH z<4(p4lLN}p(k3P*zLK}>KXk?Omrf1UB(q#IH}@;MyVVSz>uokQHI)Ex&@zKjrC37I zRBF0vX&JbSMRJ?P_VLba%A;57FG`)b`SZyr!}iMWR_$pl*dv>3V{zrL?1!q$85D^w z&HitfHo@eR#BSDq`Xq$DrRJAlH#<8!xi}Kdlbw3$odTC?mQ9aj98(*1zWLX;SNyBC z6)jeK0J0fhySDZ8>C+t@9Wf}4LPFiHzPFbZkc2N& zr#;uqf}vrfN6`zu+U>)IH2HyV2#aQ1X&L@7I+#>L~M zm?nx&e#0U^35W0f0F2lb<$z&A#_!2wPgQ;VNJ;cgwFvYgd6Wye$6TgthYmHzDpIBB z=c^4l=$E+f!)?>Ct4{|82A*DDTWw3b-Q3z!cq`1pRRbGNsa+GdLTr;b%{@f1>nrh~ z<1wE8g>eK@Q!m(39kSnOFgrC(Ki?alMbQbY$ECBAw3CO2r+F@ywiV@)VO4ta&mTG# z0j;iYzk3Rm|7ccfg+F`oB7oEH55{uf{(g$iO`A5!H(nqHo|97pJI7DC0}bhrE_(rn zCnGzX&)1K__S=V0WwCN}WpDL2=QlX3TP=*+R$9JWQqXQ`X_;(1v3@{ChE*+b4+^~V z>?k#g!#&_#5{5tT#;|f`gxh3)Spcab+p=8&0JW*a!>!1DQ7thV1%jx2sT5}!?!9^Q z#&|h4cYndkmx9nxYAI>yfuW(hu77@(_4Kf~x|Xk#`#*S4S?)_oKJK+Lb$@iHwVRvU z!oorf)^37Mrs=Dpk31i2upo7_Ey{X(4YT@CVjn(#era?j;l@BYdG-ma6DLkYMx5pK zrz8WzP;M-A{NXL*v9xW;12Egp!Qn10$D%zg2<`U>%ly<-8A=Cv*B*;Q{QSExGYl#* zmvnUA0|lY8d7ItHXEyR&sHlH`lxr2x6$|)#(?dWOVqMkZI+% zDhUpn?V?s47hkh)BqLgaVhF2mVqzj`{6pxW(f0Icw#2vB8d-@=&1EljwSI5&_#4L` z9hqhUjU}k%?=b}XJ!V5kE#7>6ca7nPJ~6pjpeXM@eNssr|B|Xl8X{nF=i1!FmdWqe zo_mo;bwq@5YtpPnsEfQw%EP>l_Ei4mW~$rW0HMqaJ916#i0QEomkM#ri36J0-%EKE z6cl89VWr4c8jxE12UEYOsOSQU#MxsqDpD@fuRC%M7T%g*Md91MXOGYF{KO$XK9wJN zAUgz>1;EoA;|uK7{Pa?+rr_q-B(~D!%a{9c#d&CY-goa()6?HWjd5Qb4)?Z4S7S)o z<1*QQr^Ufv&Ktywjgxaru5Dj6_7OD$!~JK^_A*O4ZZmS9A_s8G4#>rV+(t!JgBrYh z-#)H_PZv#1N!2`@y&~wsI{7y^Fun&|i>$kHymL^ijeCn+GIDaXj(f!^hh^9gDd-nE z?bJ-t;8IHoWDrnrbj*tJ=*+tA7Zg+lv_K8GiaD<6=+H^LU}&&g<&vJ>Q2q0x#0R3F z6Lq6`v!bDQ-jS#L21$;ey#_1&(YWP9N62qu_=hl!%C#M*@9#3{e}AVV+mbTY^1rpO46LY}l{(WGe7cU0A)|aEr4d_&)Jy-T(Wa+a)SdP#Z242nWx!TiZfsdO4qj1Ftt^AK+#xel(Y@LqkIj*WSS_r*Yi}Ok%rwdwWTJEx;$$EpcZ}%*mTK9e_=i8t{JW<-Ix^ySj z7R*AwyX&FC+XH0Av42KeBjkrT_TkBaimBy^QU*6+d38SBtoPvcYNJ#3_V%M~$>GLT zZ|>jU3a-sL8XXbAE7{mpD{*st(K6oeQ=&ygyaNpgL1B>m4}VNZf|ATa|E6QMz51CR zkIr#cVCROFGz)0i%15HJ9XxU*;k}GmO>h*`%0q*LgBh5JIPZ$8aoox?5leclbi*LP6OwAIf^yBFLxnEqCRNd|t#k{=pA^>2tx7f}8cejL;^CX$7s;bi2v+8;wz|f}>qh!Gn?FTEj zSIvOi9<%A*i_N3}@HH?#9)x-1sYSO;5c4@ms(BIPWP2F>-Wz0nL!~?P`(N6ltj2F(O0SG~T>*RN7Ohr#C_OY);WD@xyL*s&NOe3)Qz$Nv!yxlNL^NR>4; z;dbl=N9_mYLk|l2H=2-b-Z2Z>F zv!3u|hESog)RXb{Z8_>ybFtekvfCQmRgXB%Zr&aQs=SL3HZZo?!Dp<_9>H*FJpfl{ zW@g^(JFID)_{@mFx3qx;Yxx33CHjdnpMAE|I_9*6KMQ6#Zi+ez(IQmMZMZJ%tkiR1 zs}8|9Iz>otc`h@YR~sS{HGHJS#2B!8??HbsKX{9Pbh3aoQ$O2y;^rvmxcg^+d`>VZ z%o)wYH-0(Su3h74dsQ3z^IG(Y?L4B_Wc|%lzO%Ejk#2Pr&ivVf#Y~#_N!pkvX4Adf zdjAPesc$#(2dDUdetmZsQsC=p=QH%N?cd@D6!&}hfL16qtzn}KZrcuVk&avAWUN9^ z#^qWXSy|hDl~d#6)-81}&;y2M8czsl2$SDdOWjIlck5OS`v1W1-)T1Z`n+H_LAckbLV7qTS-o_7tMs}u6 zK0C4l*BrOm>grWaQf%zhMCrOI^vH#Up%BerkGWsZ&DEM48!u#)c38x$=&fEq-S+bi z%2t?JWpLt)!=;aw@<+aXJF>R=>+4Y{uP50;k4OXiaY%mo{8_#E@Ol8RCfZ=!jY5Hb zmA4L{ws9{sHuSO|sC>k_zA_WinF&?u)csv7o6o#M=^KjnT9XAaqF^@4wJ~UR(oBx6 z)H)76&XanZ`sEDBnD;k3S%;(mf?oEjoJ$~NpfEv*&T*z6T-4Omoc^VDVJjTIG9AOjud#Sw&etvj)QI~ z8~K@c=ViuRz`Xs2d}JP8dwJWgUG!NDSxNVZ(UJalbi8d#SGBTMYxa9pf_|J8XY3*5 zlTy#sC!J6y1tB-k1e%}@Xq=DHZ!w~*zO{hU$$v+uyb(u(~H0H zPSowO2l#KEBhYD+R)=LrO^u@CuX{V0P=1rj^r!tjruFdTmNUA0P+1e840ccT(W~n zbO&q^ZziRX{U`VncT--0B18=MppbQE*2dgxE!htr?nb@b3L&NZ?Uj$4yBMxqxnlC? zXWM8?T;S-=Dv+8zKUTn!ws!aQOrZ)50K$2HQ$Bxw*RK6C51@g_a)(h=2e#eh;?d}N z5PkGoFq;e86S`n?!M=OnrJHpI}rfgMLNn?(E{?AN&V6W|ms-(-yLkhDcg=b+A8Q&}6PhM}SkRAsrG^ zB5Ga!Ovp@GDv<^x0TMIC&uFs2+FzKTS@8U#XAoqH*|^K-n)-Rx`;$c=>94u2rbtxG#uYbUxA_jEpx{3J@i z&`7k`J;;bqwoR<93D>~K*EdnW%+%D>8<3K2aV0Je%r_%9m%&Oe2T};u7i}1iwgM>E z9rBlboZ%nw*LHU2*>8IPa)}&3j_`&KJg%JtR&TzH9;3P0H~ow>3K;Z`-2|mpbGvav zxWMtpMm;?}aX>rZPHt_gAXe#urjckNvn` z&bemu1MhN1zuK^211Z@em{~%tU&|BUjCN-82n!4Ev6gn7-JPPe;WoO+DgJP?rWo0P z#!}#ullb>AsoE#o%trvAl)WrzXlVXA1aiV;I|vzRju?;kcQ$!DL9NiwzaekUX1cbr zNUU}>$b>yjac3Erk2{@9^nfe2FdS~w50eZVkh?hD1teoLyuId`U!b&Hs!yIj_s7<* z`1ENf+Mx5wLZ)%Dh=+$qie7Ho`}UKB4#N7c4&tAgDHOc_L~y2AO;Y!F9xkpkBfc$~ z;|$T!oW7ACDO7q-<9(T>T{W&`{w#l_9Jh^0>ue)`mAUY~O5UDk{6wV4X46a7%g;pfpYpH)&idFqsRUFIQ$ z2PaV*DLrh!+?ZwD_k!NgwVIw$QPI3~ft>8AfYF+v_5AS9pA?v`DwP$Ro&uAJ-ePX; ztDr_y!q0a^doCriB@PY^sas7|)znxi(=Ouy8V1?#Cb0g5dxuG;6!V@|ro?ki_dC;KNRs2CU+9DlTs z6Lx2(_3te^_TC2lI%Qr9bW2Bf9ngh~OY)iRy1C{(QZpGdDN4ZQHh!>4wFv zb0UEIzrMaB_I$JVoXd-2*Ea&bK>ocj9uXWI+)?b7J4$bi6*T*Mh=Rx_kZ!%lq9r83 z&6_v(r)bDYD8~$I zYirwf?5G^738CJ%&p#{c*j`4)^3G01@N@H)*V|D`Uw^i}E`Yub$yOG=e40^dFd78G zq`A1b9@btaJ_qhsQBl!FSfa!b?Dty9RH*fR3(RB;MmYK%n>Eh;XyAMX^2 zIDTsv)TJ8W8@jGS=)h=)8z3m3#99=@a7L!{^6(^Srw3yyxAODzGalZ*6@~|f;>m)A zxU{r%)rb+Cjmc)k{jZElJpoy-`d5d(K{6n~(sKTH&*c0(q~^^4ZN`?Clou{sAPib) zH@?sy6&$|69D=7HZr4w?i&e@G4)H-rNk(v8)LKQbAi^+$Q1@ESG7u0j)aJ}L4g>#S zVki9jN_a5v)ClX~zg`xCuTK5fH*#{XxBR2t{O3pifBpK3j~pAd)W(e)m5=ihglgm8 z%hHI)YW3??>L_w?xHAM3J*dxfc_)zZDPVKeSEtDL?%n&ozJ3Q-KGD31WolcpxCNZH zYG$NSvq8jb&AoYVd3m|xvZ=hBqT&$n(=&+SFw3{0l129DY~Q)F>QEVxzpB}@xhU_-<(zZkvPyiKUC(&pBiI5~{B+|gkx$jHKK))!x zygJ_}Z{%g)t4~2ep%y{>9mvp<&&7CockpPX5Fra}ubud4V&gP45W*HNX{%O>&K;N` z*QIX(#=;Z6vtiQ~!Z08lH1yqbDk>(nwjoq93uFcbj%8WbKWA8XiAs5{=nU%v74m1l zbOFB=clf@eWPQzz&~mU4eV3$m)57Dw^6}{_RJMOLWh-lI)nqNc2m6k0d2>Fp#XRc& zO9VuG=D6EjO5Iqg;Dqd%?PxLkZuOo<&3gLusol++@`!6pRWTU#!`Q@5CvQmoY3mbD zr5P~Oxrr=MBLJk|cy_co0Tk`m__&F&F&RvI_zOG1VJp#c?0>XyWKZ0Y^j&0fn*MeM zhWobay>JS4jo4%IQjbLcy9@AAenFNXR&Z!s}o@ol5k~TpV6+9lO4!- z)Cob{dD7L{84%1f3OD|UWM7fXjNJ15d-v!+S~kqg_jy?|7nYWmR?ndI){THWih&0m zw~dzZ_y%p1dqnr`W&iH??@tbi|M~N0*vR8gnAGv(dSmlLsQ5SFYeW`p?Od4avZWh= zF{=_f;kxN;hg(ithQHT|Iu zO3ctZi?*bV>2XDh@L9gW)8Zcan(y$Y8!mM6yvu0QtLbmwE~oZ|5E}b}L))D6ku$Ge zz0x%baS6ey`)G@V!VMcuuS3T?{J0t{FYt$dByjpXLJ`vw@$p%!Ojt}p&81XuW2do( zKSIP@!FZA!(5lp9pGC=da7%guLCL$O4r^X_-$DcnnDqZvh~Z8U6G3nQc6RnBPDkXQ zzI>THBXLD|AL1YZ@`3aRd3erRuY-lZ(oEI@+@zCUgMdOb^2X?wF9X+CC%i`Cd_xSW z9wIZYe zkfM%^S996~(JL(R?2;6$xISAf9eAsmH|B!vm{w3MPrW&K?1}=U7-awz7JFWg3IG zA-fG$P}b0JUeFWBhcWURwmxYbYUF2OQ{tTltNuC|VS1psgoZpSgw@=dq)Car=37{J ze8-L*%A(>dLZYJlEemKKfxVN$A|mqe9!)GQdBmNdWIkY()*W&{;)*~ju#HG{G6)@1 zqxp(NIbcyS+LpaOi!}kqB4^O0_fus#flI_2ZD*>Sv+ zj&OHJr-09i{f#c+ZHAb@Bm_E?%>ExgE`1wI$l4?Fo5*N^O?f-PnruzEM2qt2Z6;t? zL@1Ra=k0WLb(11^8X+@G0S%fn=a!b9AOzu{<#4)&h6YTSZ+g0b3@pg7iA$!YY@RDq zimBh{CMPe}8#J9(FIb5DnrDBA5UXFD6BQGKz$!9Xwsrsewzll|;~xUf_G%e^{rXjT z9K}}LdGY{mD8>9J9B=>>at9)1GJH+n)70xeqlm?3Tic0Ka_6zhHQfJdqof;_$n-#v z)UY#xUw*X+{h**dJJAO z?jWY@_^q^>2Z}^OL|0uMg?jwCGR$x&nj2xB2@BI4Jb2K&JuPzd8DOzx77ZI8AHf3E zx_ty{$4c{p;0B=z)!>)ZFx)*l~bH-aaWXJqb|&3LZuE4i-s` zCeASBOib>^#+%@eQtk^EV;2cvqin!sE1^J?emZN5gdKYYPd3@2qA^B}U=>V5r)oBf z(>yKzAW3)gi(heTh=Nd5KSxpeqIA9GwSSB)p|q`jZ;0eEDss-0I(k%n*ah5knD9~; z5euoCmnublI1;qHwmfmx>cpZ}sxH^}Xfa#ES0jr{W9jl{zgrU@OC3I}=y>k@`QRel z4{4}m=_iluIz$HK4RGnP3&`$#Lt7@*C@0?w;C|uBT4ddxW7&yS9dOhn#x;BK7a9|4s}w!zZ+bG=7A+tkUM52E7NDo_8vME zyK(I1b*!#SC6Mo2V2}+1Nj(N@`-mXcFbcsmDq-!`9eWR~!zTHbnVIRGiK}^vPGhhJ zHd}|%$d?Xryt}GO-Ugc=ZS!eYSJy}I)?qA{M-agE`NNdYvZr~=+_mi~;MjlMDWwP$ z{{}`lAEqw)@b~FHuXTf=1ti^uf%Biv!$0}`;|Eu6R@M{bifSVxBZqNcPuIZ_N%n*> zLiNw)7nXnd@(M~Q?JOi+)ljn|0^7sj?ow3$`Q4+JH35@-vj;pp3Xh7jzpk>8xe)rp z5Tt_m)>5n@c~`2e(1#CW93M}i#`?ZEEFz+51Hk3ij*7`jO|9DBfs8~V^UD`6l3H(} zV|jbRGU3;#UknTPQ3h5gAA%MmzsxFeLvB{emv0yU)B4Q^)`2A`nQuYZ%a>zQ4G=qg z_)R6vwgOV1Pe5^Tv3cLyhZoPBKmQbO${qAriG3PE^r)1A5c%rPF0iz^%XQczaZbO% zkk2^UXr}XtiE01-^^143578C2(gtYii^wew17neduHQg5!$2=K(i&+M`nF>gBA#QC@uwN4{pM(@(4?Vadma)FE^{s95yKzDSF*4EaU zAo-^vO9dfIq#a>4%D-_1Nw(69Gb_Nrq;@0FR78&T`*px{34g2!X|!PMf9`C5gLOj6 z1S|@D^5nHa(=bR(`)2D-{dKb4d-v)jI0PE3lToC+dlVJL-|^EDP5w&G+T(brINos7 z&Nij}yrxFKg0=qP!-tC&mfd+m5X}UyH88s`4BUsOp#_Dv`JA%;F-VomQP$+d&SveYc%$L^qNTv;exjlvSFX zoNT^~X`;C4nW;-8ulXjwPhN}mfc(6m`f(8gF8EGsQ%$epdG0Q#@*d1bi zfXYN(JoP&(636{Fi&jD0-t969vOqwSK1*Ivv7)Ps3431}O)&J)BX3k{gl}lJ?qYs_ zd&4Heyz~J;*VFHFp9$)p@XpVdI3FQ?2jZsXF-0UN zslK5RGz&xumYpcKMncR~1kf&$@jkHkNcW!VOdmmWqh#Rc&CHwr#?gj4M=&5*8i8|q z*%oxdM~{+^pAF})0$|xZ4(Swd^EMHk`4kTuW*>M&tE3?S-`jw{{uFmEVDT2nK+R^5 z4i>Qar^eDtO_54M9&QVSNFOfscp_)Q!F;_Ib~>$fZgw`EH56Tv*36!7@Rbx`#l)<> zYm`x!xQbyDcN~2<=z{J@=oJHQjc|U zoBNeGeeVSf(;z0XNq~hRyw_@Zv>E#p@*p3|8Mx+<*hwhoTe1xWhjKXR7y2=Mn^}s>T zkKsFZY!8GY^58=Ip)D`OY}0Q1xtoR%Qa{iJLT_L4MJmTcOx z<2Lw;Q`z?!ME(%h*KM-7eAx#sA{%5U0GvuJc9i7X=mMr-JET&t|M!6%_Ki39g>%D( zkaczzCon!R$%FkT44V!M38_xL^9u>70kc0mdK?ZgvC42E&I^+VNhFdSZ4z;u!pSM8 z+8)V?uq%JP57qqP@89pC8q;B=$F8C65J4WuK$*s68>_xZ4y)d{@z%UBY4ejLC+pl` z*QHsmExVYHINLIbSooM!-186iM}7yRf%0f&MHsZ#Z}yTTaazFC?J1mObC0`1rHC9g zJ6T#<22}_4bZqqzoB#Cb(_a3LYIeicb=*v`FHHKDUdQA>-!0B-rR=-VS+p@ zVJ{iigK!EVIpgf+7PuyrlA4-{Qwn7xH*|Gh9FR!;m{<9q#7)>sY$Ba(i?GqT5F~E2 zT}#~NZOZ5`6L=6;_(aRQv!lZpOv<~>abRG8q&I`14paPh4n!?lj}U2QVAFQz2Uz5A zv?#l;6&0z4$+gJe_yOaESZ)1XZK!_L2z+Yx9#}&+2zx}%QQXE5Q`DmkVIyF6{KunK z2($V{&Z>^f0-A`>6RIL%hMHZTi1$vLA+7Bya-Q1g>FJqwYa&Xm5H1M%Lj2pby4*g` zq*nVOSijb(dkcjk1{^KVZope>h|ow83Ga`MVLV^Jp3_4!$$5zgom5s;9b{+U zj08SIDxJgZs47rzOkPDLy0@lHo91qKJhOl)O(h7|#rb%5bOVdaVHmPyu6Ys`Qx1c2pT zVp{_h!bLV3HqJ5R`!aVL^~OT3BGq1oI2<&H%mVjVZ)3U$#=F@q4kDxTWd8=j)&~M7 zoI{dcTG2QPO3lL5kfzNo&8LZ-(FScvJ_Mk4Qqk62SY9Sl*WO_pkXaN*-W#@&Tn*9+ zRN&m|QWGt4l*qu4nk8<^^M?~Lw15%$ufHiT7JIyiPiE2ZC^TsM)Q>fV4AM3Re*O2L zHg>q+Ad4;59T^*dqT8H#-oBOV8r#2nouz4ZsnlU)7jdoysBHV7IMe`0w0x4k0cdne z-17%r;Ef7de&G#jxI~Y#K?TWATPnogzYHrw(7N*&A&@mAD0UBWFWa$W8bggEz@Bt^ z;|$h`%26ImKi)u~NFU6{VH0Agabeq+por)W&v_L*0$TvNggIncTr~*Egk0o0>@K_S z_4|luJt4OtggBZ z!NEk-uX?2YTWK;_Ox>ox@nX`vcS(s1ye@Clb2cc2m>z-`6W#mPty|4Pa2`l~oxD^U zJ?`4sEMZAbDkaUumhJpdie(?3o0^_3M?t-qZJ`-vhIo6*0vgUuf7?IgMO+7>=2grP z&%z~VD$V2<6a?gy7Lx~ZCzU);v*?vVD!&r(AtD@!w9ybs7qVSX>t4P^o)o#!_9O9c z6}rZZI3N3S79Go9TU}jkh7c<&T-Dr6O;~GUA7^;=a?VW!d7eI9F>jMB01SaJ{7%JS zX6mBfKshj#;-9ebFlu(q`SZ#MG&x*f&q>7Vn&8KpS>az+8 zc(BtD{dZ&IJ2=fdSfzCAhElL?AhtZJyQ~!jo?cZ7vz9n|S0&2hLIsh&ITILLZMN2^ z$xv@GcX@hdMtyPohDu`6nFBd24@6gq{%B7CywQ%#L!Ju*Tf0lWjIPtkc@Fn# z@7}$OF=+deG6^M^ce?7@UE@-dNDSP*- z06OxhHwOTp@9FbvyA~Bs$K<)79&;M5BaBWHbXr>@Rp~8kSZjR9?ySU4(Uc=OR$slb z(I`3z>5giDFwmrhR5;Xjm0-fjLrW)hd1Tx4=vtfV(3Nv_rhiO`m3-*0PdQrrCOtg_Qg}eGV-q5+ zm&~qSy@xm1)_K?4TW&cpd+f Qe@ADn*EiIR1i7Dm(5EP+A7*cKB1pEtNGNW!2o} zjlJv|l_mC`$kqXx(2I-OUTN#X(#cXzD6&QZft1WwH};3>q~jGo(e~0L2_h<@aqM}= zKQWQ~_hFaV*`40$Xm0#v@aAU$+A^Ir>H>9^KgXqjI-o^BC|RPN)?60er|@9!D@`?d ztK~HKSxCgw;A~M#f|rRO$Ine0H)g#Uf)PN{(^+399Rlb_IuSeQmz<{o zoZcjgl(ejHc1%s-v@V7tfazbj&!?Zan}}R-o2MyTS*4ga|9HgjY4`|s-vRUE9!puH zVpngCcXF67e|vmEE|;gazFrQAlnkU*%Q72fmUjRc?^Zcd6i8@5K!e;-w-A!~;PB$* zO9J!JX1$4vGXX;(@f;*Oi6b-kU!@Bd7?_t4a@xH7Z5`^xGbxw13SyB*g@xsEvt9bq z){$KF`w~uFxJG333r@*n|NR>v=|36=IDoS5?oW3@KmaEJ!z>cM;RGL_uX#Wk$PI}{ zs4~EH>y|A6YYhxf>Ujc+UI*x?E*8D6gP?ls#YdbpP z{Hw8|+%ACP5<3uFk``epn$_wbSjlp>X&XDOV>P~jpy)sBK)kf#;la~;+=|b}snw$; zfJIS1K*Fk#+?tS1aLi+FqA$IbKU8TF*q0E_NP0nuYK2bLAW8HK&{L^d#)gOEMq`t% z-jQ4kY+hbk`g!>{BN+jKMuYOHTQ)cQq?G-NYq z`KOR+AkG%xE;m3}^Y*+DB_#{FMc%eEY-FqyU6j=2NwlB<*Tbl`NQE3n4C>J6VW-t= zSFe&Fx9AkRX7_jen;%JiLl#39`SeC$y|7bq z^282AN`L|>_`WnphgC?=r1ncF9L*3)OamKnJ`F2L0Y1k+t3Q}IaVp2wH2M6r?UnF1 zkmccqwo7UW!Sdg|di*E?D!fJ|%%~CpqWSv4{r$#Jtr zvO>gONma1|Y!a9RlrIQ>teXXtC4%S!MZse8PI)OPNXewsqZ(iRt4sS(dHex)gAM7LP z<>M5f3Dj?IxsP3h)E&oEX}bKYl#iG9Tvl)$qGMa&3}+8flE%hMrp3j@jWLyw15*3H z4?+vL2-O!Ml`@3T0n)fjIjoG0Z^K`I_nzrH&I5=e)YT9rO-YuKmF0sVD+<|eCZ^;Y zFEgQV$G{L~Eal9!JYNrJJAkz}RkTPbFtMIQoJpW`h15DXmwg4!1n?oH4I@UE1zHN? zkdi3Xa(&A;vAvfa2;KCGR61C==52)Y^$$tH!hD*cfGKNP`g*g zJx^h%j}9r98myB&jYP%=9S3pA<~1A-;!bRGbRFOD9>W%dlm{1w1bS674zLr)<}owG ziCT|w;$-%mrYDX^2ZUWBz@H@aB0)g+55qSeAm#twg@`nO2`8-Rg(+%xd2Fz?_MeeFB?@&8B>z1H&WxJj*TpNYf; zV7VYPhPXK_V-ofvjt<@hf`f2QoVo=EY}PC$7SyJHThe2QOnIuWl!@LcR@I8>3RYOv zu>SG-JPxGR>F%Bzo#Y5l5AeY#l>CU zgaqyrlm&EIjJ`|JjPwSBU5=2y``L(8ohQ^()RQSV4YP0Ab4uY@WTGEJI z!UAM56XTf0b$&ks2h++NftDxn?Or-MgggDDv(nH3h8W2E4qn3|okVH0eo!Zd$gZ|F zIy5Wl)%g6RF(d9Ui#>&$U9?ZHVc}bhZ5#g4S9$^fS*M&)Ad!Wss* z^s*m>N|SpFJeJB^H+naU57ay6+37Qdsk1 z$q_!eA96bf>X zrIoB+_ZBkhc0_1e&2I)pNAD*N#lc_=J9(MC8f_m8gRmOU9)H4?te4A=-zKO^?}3)X ztDB`Xl$zsMg~bTLhiJl^Yd{$lLRf=9Tla|HFK9bVWSB;|lS$A}YVK^>n&CWU8swQ< zg|pjq2rcnsTmbLP@K|;V8fur!q+j5;9U^4~5v{?=B>2BjWQqI$j;JK4#ZeG$+w9LaO2AIE? zaM|0SxtF6r+QBhLDDNQ7z7cM`z5N~BKI93%znVZEM|@t+vg=EnA#wWM?GF)(6&A=T z1at^{2%5y}*RS8Vw9wG;Xvq;LFh)l1WBJEo=qyGsPke;Bf|Ly*?c#VVH>-?$!opV^ z0h0a4{7=x&7X~T24`gSWBf;&9J zN`6@tmf>}5hPfJAQ=M^d1KRn+G>e=Pc?WNvy`t$-bX`S;WQ z*U={K_y8MsviGU^gX}*y8gh15e)Q4%P=7`?HtDX=+hHf!zR_>HyWKs>=8Ym1>c@5$ zHTd0o9=cPCO1J0 zYeVdguWYlqzhAbrny0y7jy+Dup36kXJxfSU*sOo}+x^Zome9v9{kNt!^q=f^)c1JH za99hg$%dJ`K##^|x zQ_@@4#HTlEMcLfnbmw#b*jEO+`Utw3hTcm7PwMwNCZ5=D@Uu}p>*M8o9hU-WR{5x! zpr#Yeua_?Nl#2i7quA8S8HtbjHr`DUOqX<;Ei_@gD{IM4R+yZuC_=CNV=a~1CDEvA zeJx4)@aY|7ld{7$%HyuCOOy6vJIIu3AMjZ%{MWjBO->*P<>w_)U{@DNMdnRk<3-8( ze+LdJj|=DTxMdXOBS+e58~%lZmi_Nm!ynCV!Frx3?Ig^D^In%-3R|uw{BAwJ8&=f7il1qAO=<*KdyR9=0%?BVyT zue2*T1jTYxC2g+!yuHsL&3G&>G;eX`07hXenbHB-qZ2|211&kn$~pft60QP5NbzV; zTz;!>RUo+eg4vqvrp7gM?Ga1a=)5O_DXYm$ZykPC7Oc4d>%w(;?pr_J@95Sa6-I@O#=NPuXRgFV37! zGhwVhwwf^Xg*skAE5Z1?54}kBtCth=&XLqXW@VJh19boO1g&B(>|DQu&BJv!G7cIG z+W4&c-_zV;A{Ut2^!YQx>_;qqrB?E1OkL$|bDPTSZAY}oZci@6n1($6PYv@U>|k!8 zmTs$+-+ffMHe8(FQniQ`QwcX=@KZ6flNAU)_QT~ z()`^uw*0%LSQWP%`DFA?O|SkJyY|gREgc!6VJKWN4A@up3_wyT^>@_!f0$r2F^@C=f2LY4eK9@+9ueG0F^i@ zYIHb#9=-gb61(}Al;_oEkebq_+y-%UX79wv$l=cR?rvtB>IMA}$72FWZDV^+c)H&i zVdozEyBHo8c#7{gI1!Pq+Lta#zUL=%TF!oBxsTSqj6}i@uZcJM&v%SWNL5{3KcG+P za_7kdUY!?=v8p%4cGwOFev#39l|FCd)!2!%j9JnmB6NgRMLd3pE(=->S0_jeK@R{8 z`(ZpUz~ZpS^9*2Mp0w5nWWia9tayRbX2@S{Awx*<^jIcB^9b1rWL%J~fn|u$Rdyj5^vQ9si`wI z_*1PbXbtj={$MKW$f5c5^QQM_#Z57`rmEcn#DN{W>M{?9+V5Ta812mbs^}-1gf#6n zTi^a)sk@!?J4|L6JXAk@O?yJp5a8xk&ho$+1)U6I**hMUgvJG6uPt@q+&M!2&49gC zSy>qamECz|fiTlG8tj*5uY)#{1N_Pv8^=_SXyIfw11QZVGQ>*_#-{Ml5~QtoW*2b; zCgApMvZ@*IcjB2;P-rEHhcn>7My%}JEs)IK)e*;Ug&>jP6$6Zj+_vkhp@XjB#a)T! zC@Qnl>Dhiu*N(TavA$36UdYv-_KDW@pB!30eO_70$|A?->__U1*_GY?RQ&zDohE)o zd76TpM+Z1;_{IYKZtoO4mU{<7W2{|DdE;Hn^iU_&@?Z1~?5Eiun7yvx9| zgI6iOXxv_oxnM1W{6g&D*&}an;Q=Z;uQo;n?K|4r%ISfp5txSgJ%eXQNb5bzk4fu-+NUUzS5X%=^zt`z?JAiM8nSwVPO>v)Hjto?T;m zPp)sbiDLMH<(m&$o^4y-`Qc2^eZ6J>gqcNAXUE7p+{GpGt3M83e5FhKT2bcIN~(I* z0tIm@h|s?4!fFLq!cT-geDO+keWBHHhh8y5zVIa>#W@=$;mdx5edMW<&vAUF97i}o zqT%I@9lHk-z2Qjr&nJTjko^#$nmXX+_n`l6KL-kVElk9dx ztIsI~;?W+&R(>~gD=Nw0`E*VVwf$u2Ox80tZYG>9qFeU_!8DJCYD1g^ApG=_#qUg! zH5Z>OaaHk6tg8O$y^OE%Z zDqmTY%9D}}|IRw*-5ExU!qfz(ijUEah@;nb5gQQV4heRbF0+fDt>$(>9z+7{~P$CwTt zFS)1xBB;(FzU+O9cZ>a@j{%d#{wJ)OsPb_Of4aFn`7c z3yY-c8F^$Zkqw9ZDhaKgon|{+F;c4{#BO30D96EPrh^wvNiCuJwF0Oqbk~O~`P;rq z?Bp^LyO?QsIBy|{X14a6v;Tdj6L)zJQ;I02-;aAEaQW*#rMz3aOcd!44fIMZoa^5C zIKU=ulH%V1txG3jl{ltLs=>2g?7=(t?p?W+d<^#j<-`YiC+KbFF`JJ|=8!kY^`5{Btjaloe>tlW>PD){*Y8oB6In9C^X&;hD|%uVW;dMSv3j zNBIxI*ZoRwQqYY#`LxHxk=74RU!Ld1_EzXg-&N5aSN&3fmQvi~7}?thULTgV4FU}- z0Y5i+aS!|lp=p$$%Mp(SBa2B-r$Rg(M;KvW?joKzgrkvRO9F5rpeQLKaMBF5#nk@? z>{;w0@b67{T4CTY^9#^+jaDV}e!>5TwD*q3x_$r0FO`ONp%N)2w@YYH*)18_E+nf+ z8D*3$(jtUXM#|nhWzR~8k{t~iLI{yPzt0oh_viEd{eFM_u1Ami-Ma_Z^}1fK^E}Su zIG*E}GwlmPWCugK_|1u&++V<@XYIOPzC2?5Mj&E@j->RGTi;Y3e(jyQ_G&lduP)oe zLx)c+D)~vb^}s%+)KJ|U*B%aCEq`!@N;5@n?P#CcWH6VWM%2c3zwE(Tb=qPZ!`r^c zdnp&Y)f^+lk_*N43m%a9Wk$Y7|gxP19CX^1`1QJ?MrXV{|yh99R&qC9&Ft-{e^ zfgY+ldj>sn*jakP%IbM(>G9vw;Gk1nfRC~6+rQrxtdDp4JE^=5#Gg>vvfQtnaptf+ z?~hyWIs#;#_1nEU`|67@{lsE1C8uhYnzIbQclIyHmq~gZ_2GJ}P`o>Z;!ZcydVOwP zq&I$yrRHn@N@r%v6`lF*(R|R6XHl*8LxtJPw)nGwtlw2$<6);}*tgZFPTeyxGFzxxmT>n?l%2i0 zy-*SsJO7MBojT5m;8Nl_iC5neCo!=L(-Hg^c;y7U^lDzN(AqhC!4<4>lSv!nAhLrkeofBKCcKE82R zXCGa@jE9}oBRYy*uK5<JCDwJ+|Gs;a+lG67qYOK3dgJOH#3LqX$HKwJ_*hb#P$OBP=pQ|&&>KxsM`6n095K6?NCJKC>wC6-clXU0cK$DQTO zUqq3vTi36$+Ri74 z6!bI{)4Gh0_&w2UbwQ)5K3Qu6kQ2#}K15g;x&k?Y7cO!)Npbd? zi92M8Y3!d^;`d*-h1zy;qydH;!Ed=h0A*uK&V-+6io!doLGk;qmw#MyR~kYsI+q0_ zzB{PA4P3hYm>k1w?u_NLVC+GIW^rKtni`jKd ztgLsyX#v#wu$(bs?`@QB$~pSRCU=qs8PC2{iWd`-zrwGq+S>SJx4e?czUqChu}9Za zTpnF4(08BDm2bC5Ko3Du_A6Jez(AFQx(LBHPTN?ea#Q6|pVQ8=o2A23jf$p{?i*b- zD5hDNZ?k;EkNbE$cV~q6 znH8rtOVq_(RrjXX_f1&(abajrt+mnIZ{fc*S5lR?xyl8teB(AjM{$zteM!k-@p@Cf zHs#ndhUOdE4{5Reb6WuaRsKLkCVrduTw8K4m~k1!qFS-e$WK;s`MzD7@+phQOJ#ew zAEw!Dv`jfk$USqfh^KDkxo$XHdm&#mOCkPZ*x1wpjdxSab5YGyVU#g zC;q>`!BIg@PCxschw*mVh8$1VZ-4Od;@#_P!PE$y<}N((R`okSJrWd{=qM`_S1_^` zjZIRXMsF%vH@{hNrGC6f4wWBr-G6h>O5)I9U|ZzEreR6`D=JkdEPlQF^G``pcec%S z_i(@3{|LWw`NF8XHl7%E6fT@UO#6X z5_4EKzDT?|{?XRCOqV3Pat3583#5JIHg~i)W9!QOu>7OBE7g4M&di?1;DEAw_ky+4 z|Hse2o&EXrz~?uv<5%7&+;Ndo2@9LDJj<%G?we#e^ z_lNonX;8%86Zadq?bq>Lyy=#lqw+|5)#rB_Qj&UAl){jyMFu%N8_0K=eUXQ3=)W^j zb>J-5@vzuRp2LpQ!cFp&?mphaX~gyS?IT8rgBES4@iSpg#n$wzr(Lw8 zGqbwm>^>eij6Pm8a97vOXRb0+75@GP2bF#Hf@B`yg5;jAulYFoP({naD2lk3ah!TQ z&5||s@6S4<&R&kf8THs7od%aQjQuM9G#%R!6~i-R{Lj-K4WnO7uL_-a{?F@KNUoSnWxdooW_!hRJ8=@oYtIaTU(|Kt5gJ;kSS{C(V+HHq~d>VX4PKiU1(O68?eyBmk9 zn{%DE%`f&HvK;pq$B9n^1^H3nPfHNjAa(j4=_CFU+Fh8c#ae0ulb@FZZi7n$m!lAE?NY!{0TKS1M?bQUPPi;a{Mh8xPDKX4O`c@2gD5UIhn$LCuI==-G z+cwx@ZZL0lESc@e)~eq!ktbEw;fVt4;(M$EsVa|5KUFE!#PU+M;@B~@2`z38E0c9H zb3GoW7Mvh^5u4*xVwi~R#cPVT5eCiXR{iV#Cg>E1wIX^z4MUp1^TY>#39GEAkgW+p z-U6%#J_7WhD!hO$+k;RHP&k{k1_@hJVMQb8w24LJ%II!N7*O+5J`HtsHwp@zt~oIX z!(ebJ(cAxl91J}P2=x{LrznBIVz+sVXoe-A4FjZ_K})7r-7i1e1e<3h9MMm!h-m8W z-K`Eu=$}V4o=1G=xE3%I^p=07ibL^?^Kn9kp0&{c6XOi%x_EoYaJO#VN`Z#;IdIai zK&c0;efN?G`gwPzHB-7FXFknAx2%%R?Tc*OA~y!pG}3-{yZ+!l*IJp8-_&obs2>wUWu0aZWxx$8%J`*(1bC@#RPOvPH@tpFm*E%B>> zA`j{Z5W$escG2R(xt{8CXvFwM$I7 zBs@NB>f=7<*tw_?VO!;S^U+fG+_0g@-7(IQYkbzd?26j^*EN)%y}$iX(ulEMP^qaZ|wu-6>ewb6dQHYNCIzQeICVAMC18P2oC3gl{un}m8gbkgh zU&26nPU?rgq6(41yC$TK;Jvt|EU729ddRIZZs3Fd!8@a?LZ@t$()*sFmKB~}?ZDgO z>Y$f$EV|}P}#MTd8=QzEDhENPqEKmCa%J8|_hKQXL-Rm{nb%W=3#K_#A$*j8ywmqE~w*)rJQVI5gn=3mOSdK^v zJifSV`LBt(7nEHeZMUSQs@AmMFd6-If2R?xlIgs;TUev-#nGu>f-eGpXYWX#96m7V zFrY_E7bxB>p*Luvvcy>P!^!Ls*4)YLO`ARlNjkIbWNJuN+OuEaP^IikHLCt6=P`%j zTOr5v;(50X35AsGHl1(2(6(gfPTTSmp-O}kNfQaY3F_-;`{w3Gbza%22jBfirsEvU z;mSDcs@!%7z0Jbsymb}6`4Iz_4a6y0_Vb~SF+I(^E=-(whg`jmgq^O7ZJ4z9CfpOQ$?ifRlux#gkF=LWG= zdcz;97r_>7PU_}}!AG6i%mYt86pnB`vH9~=>*MiLAD;!sb^Y^s)t$d(z5iqe|0|Dc z=g*!cN_2_~ZUS63H*#~o)Zt_32G^Q2?;ZyS*TgAr53Wq)vG4O~a$HCuG$LU2#AusX z+@&jO{^;%X235Wlt%*@Y5(IPu*?EDO`=ZJW>8?1Ad(x*b zsLE7N+pzgLgz#M#KX7#WG1JKU%YDlhIvGS~K6ral^v*xc=Irx_j`Fv&=g+sI9Tx~% z0<-|jiKCn_Xx9<-eTau}?IqT(Y4G&dcjx2}S3%Da#x+ou(~AR=7YsqVyntw8&_UUD zzkDn56~hX~XmONpq$LEQ0v12W9y&iqzo-G{$Jxqc0Y7ch8Km4rEC!$8FsKJ04e(?% z35;!OBa{bXYskjtmwC1uQhEPfnh`Y>%;T$zbmfl;_jB+q+tBCs^6l$2?DiV!!o!lW zF{B3|!M5VL#OmU&t-W_xjrkVzib={{V)6I+$4_XQciZ+;LYDV8&8+k99b&99 z&2B1wRZV#)q#~(x#ANZyR^P8?=8*#L>t8d7o9V1!&D4t*`~5+o-e&3;*Pc1Xz*^;g z7Ra|M95&!2J9y~_4%X>BwIH0EpgA1@UldEc#CJE=^r6$Ix57NF8i(MKs#drYrdvPE zL`(_0WPK+*e4F$1*!j`1G46~RIOY)dMZ$4MixifN(-1D#4X0^rxHfD@C@l? zTWlk|8$z)G;j(019d7K_fH&lu!A2;NT}cjXHl{>Iv^xn~W`4>FBG&H#m6n%N@QfvF z*wbgugr^$@@kPhPa0ml%q=yzEbp)itt*-u-V8S`54556k^XoL`$qLt zNi(@0TL4Wa;SOPJ%r=LSUN1s5hW6libFO6K#mU0csH;47XzVP$`DOLDQ?Hi~8qt=o z7rwIN=lU%HH@ESOY~82jp`*Ga0@z#B`HrvkUQ&Ts9lI3hC0@D|xn{P~blLcO%brE+kMf*tpB#}A-p566tIO(^dOnn+Om4ntC^3a; zCre{KF(CZ^(f%{HS#3A7zCyqzw8NNuQN-x!l_N{oJe2tvdobmBq}o z(T$l@>^0e^_2IsGu{GWJ8?zJLr=5Nu)D^}kz<^>y@{)NnhS$)=bBm z@36WcsQyK6zu1{D{z++>%GWR3q^iysm?cQZ{21w<*$`R8#vFe?TaIdyc_g|}`beAl z$sK3+_RoY}JZX40tf%+Qdg&_JT5;*WiCAhdoSD$wHEPKhx^Tgx&|nEu=(eM-gc*u| zrN}zxrGDoKc*9hlmj7x)JmE$d^J)coDw_^}lm5DJ&~%*c{hdxaC$WB?`@d$sJlo5; zZ}|t6Z_|GL+?|w`dchj{G$!f?cVI>js&pFCICjb1dGJwX-?M%JVO=X-unV3DSSp#^ zZ$I~~{m%`_JArdg;C2fqq+#fgQ`y1k^(aKbqnd82nk_!pU;KG+{QCydU!2$nc_GCm zFE1}5IVnYfjshb#!kC6*3=^qXaO?-zNSm}>%u~OMK*sISkVrLBJc2cJPyOkosEvgV z{Mb3AU)zVCn9vUV>|B;LR$*XzW-KB9Cgxpkb@GIB3 z=WkEN%k$mm;Ng~ymseE0p1bAU_5K&s%&FSOeTQyPr|5ecLwa^yc{0DP23zjQM#H0Z z5#@m$?<(qJ{=(4(2)@P0UQUQGdJ*m+ahwK?@b^IUz&l=o*8ue+YCp;$@XkF8nC0yW zv*zad16#iT@RFjq1pW}9JRgcp-QDkV;EVs0jNQf8>3S%Xy3CO0dRd`i5w|uyzOXCp zSMrW!lq=fBZ0wD{l|*NptGPDj>mP6ycUZQe_g2JVL5;KR(w5W1>+^3by2jq%-DzWz z@T}3`g4VLy!5z_=NjEpRSPMpH;&dPF_;xuu^HKQPpN)eHYwS0xZ(1}nk?8jktxc&* z4vf@5Ato`4x%CurzYS zfSOnU)sW_{K>$9%^L8)ksl9CeZS-rbBU9xQ8YrS9G0Rp<3X_hzS+D(c8RT{sa)^mp2DB?j7)FktcD%Paoy7< z7M=NaG@$w_PdJCVlw<<~6Xkch`3jNcim#$wsULq_+5~b~gmsUMo51DO+DEd_ZB?I`epnyG){KcJD#B;uzCSvE9Pn_;MBF*R;|Oj43CLy0c>_wPeU zL4X076Z4Vr@!SutGfvwOF26Gcy3_7Tw_NCZb;7Mh?M#_J$IBUXpBghmUnG5trBL8n z1?%hMD`RIO>Z_1bC^pJzXx6n=4}CajJ+UNuzv4t`V9k(6%1f?ww>>=<{{E72g2wzi z_dgEaY#UfvIaLd)wwrEl+o1x)2^7wEbURhae)27Er(*+wa@)M!rO%ANy%}&lv%=>6 zlME%@+Q5g~ONwuwJp4t}D`C~T^joF6lE1po-@5(a?_T8sbP0QWDI6aU<+f38koMT6 zvXeW)eK~!V0BOZ>J?&|%T#z|+DJ|YhcE9-b;pgY`((m~G{rv;;&0(Z09kr`4X5t#2 zH`R7FgweZr+>||g?azTE4(;f5qZhccUi_t9E&wxiDGnDYkhigu=9|i&_hiN|nw*(- zx4)!YcEtZa=TEJwV$EvxYCKsZB}11m-o2gTx|}UFd;iv=5N7hsUb0lx^QFhtS6#R1 z?EdGlmeV~Z>T2DRt^1@BE;);wSWUAuq8hEflB@bg`oA`4l3m|=YG-FB^oTpiL>D6O zMSC6Qf%t=K1pe^Mt5#M5vmy*f$Kp9!Dhax?6Mr(-5JfW>@Jqmuh56QV zs2M`93@n3SMlWvEy<%aSFrsMcqKn?|{Vg)nXo#2<76-Hguau|WYs%CM&)atYP5Pa` zPXz3g1vhF}g{8wPXC>I$wI8m)Jdg~NDV>5CjOb>_oDK?-7~-Q$LJ)LUFX6{YSiZ3R za`%=%z#1R!@q`GF;^ODW1hoT^Uy@-m5T`tc_I0bh>BRPkfp<|6n^lh7cOqd+EMux}+Fy55M?@CfVss>HnikA2HBbpcn_%FDp?)q4FZG;jq4;Aj`QGWct6ECy6~p z&@y7%lRO&mlG2VdckbNT4T|!vlTRp=Ct%^Yu5I-*l+k;}Ror056gQ3KqNJyZvIzacTPP@v1c=8o1x^o(pTid0;tJ+gz{md(to3127H)<1n7s1# zHYnR*NtA{=S<$ z-PU%b{~Hgm`mT?Nhes?2D!x(Z(hj2jPSslt2bv56fP-)pv!D<|Q^^Se-2*$Mxa>Rs z?jJTxVOh6#?_LI0SUz7vG3?>ZMeTc~5qF!5c43#?H&jmbI|CMGg>KLeY?-T|7WFh{ z|JV6f&MCg>v1%8vcFd}Jj^zQjrW0PJm;$EJ-v=RN+^+!JUu2RI3n7hoednO^11CKc zVIEh%^NrkLvvVFsGvgs28OtsVE+_c4x|I6ZCc$EBCo`7IV(zGQ1}j*8Z$V@u^8 z=PzJ!7|{$ISOIYKHqx2%kzfm){_gAAI72 z(l=SQoF(*XlJKRR{O7IqJSQO*1fN#-uP^Wl&7(2@e}2y`_@1@% zj0U)f5R?LFf$Zvd8pBulOB|`ZaGl!IQqopr{;tSjNVd&>z?%>pGbbt}#P{rhEo*|O zUWeE0Q@`b&ynC(h{is|%*)!d7n4?zQ_^c=lj44RojJhhGzY`aLmdcF;hQf>~@IL2rxZZqCoI@dX#Qgu3lWLv;^Fx)zcgCKY8g#Yuo|0VA#zOj(f111z-~O zd>RamK#8m3(8~D$G54D?Ysq&9dC>seq}-J!-71dUktD#uXioc8hy-6iRfvvh38=1w z&IE77xZo;G%diqZapDA-X|sN(9BFoEc%b+0?O|`__r$};hcG{|8}GqbswNOt1)hcb z)qDUVMEBdvWM#3#q>uP#Li6?nJ`x$x+Xi3_WqYk-7k{7tN5n5Z*d zgm;_rodD?_gx%wEV8f#(dh(z+Kz4=m#zy!Y)$RdMHrP8Avt}5)t9y@k?Q$-FT!~`d z6-&siOHza4F6`NzbuA?AA%M&vug4Eg)=X#ch`cPjp>CinG4?J7ww#O-S^&no$Y_5pK~1PptLXz@Dz^7@@LR6oBJ zxOfD-YUYeiJ@_vNNMeeUg_YG`6(r`;Vf2y#+G?ZVNs&eeWX9Wg<)^xM5UWeHLG8fl zR1Cv)GHrlNw}btE^*=GVg^82;j><6FUIBq695}sH%9qHaTIALZO>jf%C*21S{8=bm4Zo z;!Gs%Cj8nWdU`s_(`V1f6TdRilSOeMz{;C9Cw4o-+3coB&eUC|l*hxn|5*#c0{)YwhM2tx5g@D_wB zY7GD;I2M3kq>tN5{H4&yf8sD?LMH40Goydk01aJ)~?g1t}?p_$-89HCEtZJRUOn=fdagwd>rY7U@BFT@n?sdtvCq zt%aB^)aA(-v-Vd8C1JC z(bmziM6qh7D;P|9YJ=VD#A^h^$ACGCxyiaB9SvfOe?14&qlh;fnR1G0H!kpILvQIi z>@~h&=|%MaNL{;t0&V;Y&=J42w-aNn?A%=9WMDVc<##%1{~i;S2jYMfKnozYs$i#+ zBpuHh2tN}w%dR5v6Dk`z3b;f^Aln3+bOqjn7Oz4Nk#Y1#rtP_8sHj}vZaT_r^&n!a z&@J>a^Wt)h^+B#{ANLd?mkhlmW+=$m7&bg%Aaf~Conm1*V77vbi!1n@4w<$}o-$=C z+H0__V(bZk>nJdCf*wJ;KjCP}9<+=pF4fI(vA#@um3c_bE3pwkD^pnr&(>Y zPvA)FP3ardm*$ekIK`eP-n<^*Jx`(ORG9`5qyk(F#NuC=y+C1fe}JSJ$K0%c2WGC^ zNrQtBd=DlV3E;BaA0 zaC0U4(~M6SF(4}v9eh%|AY%g#LZsu9sBQ@Jd24*c#XIzxg;#{IR@2~6jUvDJ)vHC2 zVi2D+GVE3W_42)Y^jCAN;CMs?CBNYcfoYTrk!9kOu-AoV;ilSU`XJ@61c4TKnKRD{ zjm85bUhx0sbr40>h;4Zj0}+tYr~bKlxWz`VPk*XAe`K-J)HDfPw=tWM*u;aN^?+s+ zT`?!IQ^2@lrt~Ax$2pe-Ze-cIucIjXj=S-ur9LOF=M{{fI>yBNS-IhxfvDG*6oEwe zvelCkC4bDGddFxXI@1+-w09F1zG#bnI*@g;s%G%2kCk`_(ny#X-h@z=yGwv9It0vO zgg9d9^&)eGgk_Ll^!;QZqR_XNbcXXR*2g}0j?f)AkhP#_s}ttN0f$}$kPw4*SHvg; zGoT3j+1ln;@^lbxD}LSU<)s?o>0tQLkh>u zX_)SwwiVS)G2r}VU*#*k!B8!(t6PubkzgofNH0#^NObu|VfG~H`LL|Cl+dX+?!T~+ zl%|-$2l251AcH_qP>A-I^MwCtoNBi`lvN3323G6DK_3f-(ZoA{WDkGNa3(ppF*6ZF zfpy82I*d3@aqiM9je@z)JtMetc>1xJBQwMzq~#9%{@ZnTlB_JDgZ4B1E=aq1;{2bD zKfA*f48N^=^N58xX2~nhx+UVD=pSmw^s~-AgJXlC`7mDqi%B`FCm@Be4RF&4a)fo@ zgWE*)T?NWWDdEuIVlY)kAD|JnBVHqQ3%v8=M%9a;*+4tijmV!Vh=+v>y44xqqYXw! zK|)c6XUSB)6K+Z5d5GBjqEIk$Xw#C_9;xSK6&1@+wi2^F9*^Kh@QQeOb6Ma)OAgBE z#uVRpF0HKINBpU-Zf>Si&cqH29b<~iPWhmv-t3^F5=A*$AdG#F_;f%bz{v|lh+5m_ zjTXks@^YEBNZ<^os>a5LOiMB_1%O9-Comln1Cb}AWg>>soq_GD<}6FkbZ4gDqKld` z_wKf}$FZu`3sc!=f&Q`0tjmnQ*)cn5whGO6wSIabka2hU5S~LP-e>BxiHQdkPTvRG zzE92cBCmW5!%O1fFwUkw;46A{YG?yIxFsj9{HZ(;p*7W!OoPSdA+z$CUwNQ z=_5-Twcp9bMOa&q`W53rq%@t93qTi>+ST;{JR^AYCtficDEuKKD@#<=#6J1hvHY=1 zKqBtbH+F&G@_TaZGHm{~z$wr29yAoQ9A(q-()Qg?p4<`;{KCFEXT=8@y%XauR#A3i zJ{G#`YQs3_*|1Mye$MO%O$*{0`N5OZqeyJeoI9s}^R_jiB~wk4P(`1{-+)Hvs!V9; zIROG%dIh;rM52b(VU1;e)zwNh$-e~Dlym}7|1yGhqi&LiB#2~kG3kf+_QpTdb=^yF1MYnn+!MO@cjh!$ToWB!( zQrNL5tru_o+{;siaT`y-{2>_^F;hr5l(&DNFpd9yIir?je58q=E37b#@~*wOnq_+V z60I*qc_S9}Y=@ZOm%jou?ue$&e4>@CVlK1USJ>>HJvG7|M-fy)+)Q>^N?aLT13y&Q1}zjH{~4(-S& zXU?n>Nlg~_o(cB z%6QTE;Rm~QOiTDTIKn@!v0%vx{zqH64ye}n2W@g&k~)2aEzA33(b9Rcgoa0?g(FOa z)2RpbqN|P^bgI|uu>4tzRv#HRN_md(y9ewLKPOdH)ifj&Xq-Gfsb8*TuMba7_-hxx zdzW(Z-O->t>uzQWCg|#LKfJGpqJ&9CNS=7LQ%ExpLP|37nyo5t6}BN$87fA}EG1P= zb8A`9E7Ve~sN6U|SVnY^pu$|bI$o7)*6IVIs3}{1UNJG^J%`nYB+LEXcixM$&Ge8a z*$3#QQzyDRWt19qvU@b1Fn@A>$fEh8EIAR)0L^kn!?|{L#E7R{kXJy*dt<@&wjArb zGS57@IhtSd9M{vH%781vqOX++x6{+p^+gmd%Rf2n;^R~3EFJkYuBUqL*;@#+fz ztM|1+{iLT0KiG9KP2Xbj-MNQ33TP;WOs>E|7x!xfL>D>=%l-=|<1nxkp-VR{ zxe2ojP8`_Ci?G!S4hflJkD}{@2y5-Sx%KC{gT`b0B(Ffs``d54-dJ1t7jOC&IxMY9L2R>d4D3Gef8+F%2x|RN6M=651KwtnY zMr&qUnb}D!eU-?+8J&=wZNp?QNQi2eXK$R9Erk4J@H?`D)_Y58yA*ZOXQkSTm2yg7 z3bMk2KS35xUibK-|K%0ej-S!*_IU7QbM(}@V{M$rG5Ky4&fknFI0~Ub4#H}b3OD6B zT!-M>w17hRLp9UHm5xFovtZB`NDw6>f5EvU7Jm^q_78$7yLaDAV(2&$(+i(6R}yOw z>@Wd4zo0sS((6zYZKM!h4vu}Pv|Fkuqe$;rIzr&$M_S^J!!l&ZGFsu)D6B}SMS%tA z4V}D-_fGy#bDK95yoeYiapl9oh3mu)@u{Y_MU`$pz{56PdM?7?GgbW%xcRSQr?bQ~ z&UrJg)+>1~6;dIEqJh$#NSFNe{NXload6=8Tb)3AOjG5pgd1TASsgJ!H+p8J5XxT_k

P zR_{1t;DZ;Ob`N~ExB*kl9Wu7f)&ae_-Fz}`o-9}_?mHDjlE9oZ_d?zt;r2wEw?nsLx2L0)E!Y(_Zq9A|rw?Z^(LxN~zm^|&h z7@&PgMs02K)2Ax%hg2(blut5GdQ^>7bw})M#e~ zR%1DnGW{oX#t7Mlu4ZaZWY~hQH_tzt;Mq!TvGTv5*}LV|E8G5qrMp)Q$zOwP5S+Rw6MoSJHBvgdQ&_f8)(C#|v$nEl&qloy?oMLy z>cngsWDVh{AAdI6eo3arO?K^cSTX3(n{n^aBdsB8C8fLIku$Ngr(zi6 zPoxWt@V;SUW4n09)p{S8;(q@ACScTiczF#~JVu9Av{y`O!?==!YhsGdo$3plT(`yd zu6#n_xiH?RR`Dh0CGY#G@dHeCmuVRWAyb#`=}tSuYyC=CJ2hCFkukCIx}x*cBWAN{ ziIEH~pBkNxNu@;l>MLwxp^?b%JonL=cy}<%Gga{gQA2bD)I{@)JnYPrHIVmYko#3I zSbHM3BgFAa;k5hcukrC`CJu2FBW6*-82&fnHtc@RTgKy(l1u;0y<3<+ z4Fw}DrqVFw@3ihu8c=8V+g~Hrz!&4gU1-vnbgGJe?gyWb{6p^3WU*x^&=oZK|lZ|NyK_w$q_sl)gtr<*Pec-}cs^2Yn}FOASu z?c0l-zXT*-_kDV*WA|?3>)e_A=81bRy>I+`dntvI`D<_|U97&7>?!p!WO7B-|7N5>p2 zbDsYVuk~}Ea9!3|VcfEeetaTcD$hS=^b|k)Lg-dwYfX%UOAYEY*^#;(L06 zxW8(RfrY*FekuLinbxxQF<(B&lWsP$!8&C9{I@L*3tJGT*`Iwhda;zVOv9jZywu^6 zbI(jqf45V;Qlj8JwP5ZCSvJpIwS4EBKhJ(7?IQP3g!Vr6Tt(ZZ2BA>0Yd(hLDA|`h z{P(?GM#wWuYMNZw#@_N$?B}KUT_d9+^0^tt;a@2dStreu-EG(p(mJjPm#$Rt{H@U2 z`8rySJGB1auhKHRIo31Qy<<$2VP}%aljq`m_2~OhXQci{*TtOj0wqGj%jYi7NSi!o zn*|0RgY=^XFUa-vh*Nz#42o>nwd1G8iW*II=WkETQL=N^Gj^YGlU=#FPNPvOTKtsM zkWyZtI6{WbzHUMO`Je0y$DR4G?2fUyhhKYCqtOK{>X_kf+w&FQtlsm)9+qj>q4L!GZZ%Emd)Rc})?6mIzL4Ah?($fRyR?^Mt}A2GhN zk*y$dEhMP6@%L-~N0q;`PZ;W$OpV9Ay2%-LIWatD)I&!gc77YMEF@1M`1xzQ@^iMH zYRv4uk?Bv4w@%nmPK-u z4dE_7RXdC2Lq?AN+=addMe|?%4SCeN?7JdW>e#3~`rDZ{Z28X2lW1l1DP(HsT{q)u z1U`j_yl&4T{`&-x*-r>-%I#Cyv*M_eaFA_$($EohKXEob-y*Tsyy`UnamHnFhyZEIfG*M6vT>CTRw*CKl>oK)j0ouKEe|xPbrN7UJ*bAJ%!w9 zuZ|Dy7+a?Eiav-_>5PfSf4<;HUMkrW#)Gi)^M^*=O$7beHN=cPu?~v}?!f-#T>clwoUK94~3dH8R##o1hbFXXrft zIfCbn4;zjRwE0F|ekWAMD~?XL5bG-9*^P9H27w)4-mkqv!3t9&MKc*3Y80-m2w^6d ze+29p`cog+a>9v5atDR>_CX%o%EA9u2f2K-B@MGaHKYVjxCJ5q_B*rD0p%VO3kwyt zLt=m4?(>a8d-mi#s>}Mjv0JEZIS(%hu?dDmZkvdc4)K}#=wuQ7bil^$;>!7JEyJ~P zf<%B?Z10Lls=AU=7#K~rTwP7zccIn~PfhNB{*m6aDPy9?k`m{wrxZAOB;vC^-}&f@ zRXvIQtW%=Qw0v^u8Qxvt!+V_M`_+D_%rDC;6}8u8k!}tQ3}EhGf<7v3bO3APV?(Hq zD#Z$r&FaN2{kmww#=7^6iSh>HEo|QnMH2qmC1bo~b?0ek<$}v1FYD6ucx);y9Qzi{ zU2yf2UA%Ety`MD|VpY))Pw{X=Ygw=Z+mG?{d4W_rns-_z=!v-y8Y#y{8jR|K+;+oQB=1SK8if z`rq%HLl!NeFGNLQFs=+KanzKR!(kGnjjGiqZ{+ICL}*zs4@HUnaM}=m+~CEprGvg! z9wZ}*e(PJ;HrVwyQ4@WZ9=48&DH=_D(}5r5gzmX`^_HaA5C>Vpp8Vw0*4F0GS1^LF z0d(ab&!_Yj91OE($GOIou-eo}x9hSABi@s~QPXVv_bY#2_H1J|G&=u&Z)(clJF%~* z^-Jv&fMSUdcW(I;0zW5od#j{2(HSX-_9$^z%A_{1ihas@P}Cg5P{y7o`E(T%b>9KK z}HC;28uhF)l3Zc|a${We4}}m}#$0v0-JI zelvEztuq!wGTGIPdO4;s|NFwonnBCh7`jJ?Uuk9aS>}3pB*qX4!T{CxRJu_(KB3F& zKudLq7&es!B`$I6YI?ugQ_QsS9hv(}{-!egii;f%4x3K(rqa}ve>7On-@xoY#)sQ( zqaf}MciU2nZU@S>7<6uQ3`*~Swp$Bb!%!mz2!IvHB4Bhk#G#8DJ&$q^0Xl*8EluII z$@0tTmyG6=T=JBC;IK7IH)l$7{#3y+xvSQ7&^$kP)%<7K`ZLnJ(N`8}w)u21NaAYY zD%OKxoA#YE`kZaauzGbPK79zr>4|`jVE%T$w_rq<)Sn8mbzSLnj}M8$p}m<;2%~B zP{G3@=8z7mQFM1oo&vgH8N!AXE|=#`!!ai>+Qvz%Ncebl7JQPD^j&O!@bAf$U&uW7 z87{>vYKPm5KsXgfd>r~RGpzw!;#AJ1=xIzP#he9QQr4jZG*o$j)pB9jv}mtVxkBoY zsFScmQ@C$eGs#Z`oo{m7Xl}fOymjZ$<e#K*V9)sAFr4`@QlmlS_N)lMhTAFY>uQIEPX-^_Bk~kEN@uJedO2evZ8T z<-I8U`Sm)SU7`j--2pwdaSU-Oeflf z{l%?mdZq=R19g3cU#+Bh-hOAcCqVmD@PK<`<9yczte61u;7Ih|eT0h>7K21Xi7*q#ECMq*4mH;=j0jimY1Q{cQmGi@4@F49hV z4>*#%8Ftvi-W(wBLtTlf6;)M=WSYqIKtR!KpbpNmiL*123w`V#Js>L;JRt1oK!a0CVDUwE3p+J>Ekx%AePq0(y)wa76W3hOdj-k-KZ!98X@p-t~-~ z=y2jTmbw$Q&yW3aG*EBZAALUc;@zS7wcTDZaNT2vvESeJ zn57}_H2mcgBg#21VEpB>djx%=NHVGTRJKJp;$9*imbCBp)>c8dxi{OjdLar8T3&u#QBjL$mP$4h z-hlMlFg?VoQn2wS5GRS=s*7MVI6yi;=8a>SQ^B1C2e488Ky%ay{(|&&i4|cF|De)c z<<+Tm+%+DnX!yqujkzqnr;c0aV+vb>!0;#etghhpmn7!H;^RSZOu4-wVGwPyaZ$t$xb*efX2 zT0sAO373$Gkr4y54AJF(C_eFPpA%9ahNVk?9|Y~As1f7eXxiGlWIS%)R>EXcGC%-a z$nM%)=lV~abZwZzeK~qR!T4u_&cA2M zt9r+#=J&&)%CRIjkGjsmk8eH^Lkl=THe*t8kkI8A+-e_@(r*^PV2y!yj{~+y$9OCz zCZ>469_8ibxj!R3cgP^U(ap%yMp|iwy9{d~EqKs3!pKt%*=s}uN5S-XH=#T19)afBrNg>q}zt_l=ao zagz+=%h}5^D$l86UcwM;;7u@r$u|mJPBnCWV}x|C^h+YeQiN<}95-Ij;_D$96%-_9 zMN>9n!%YUhQDoDBrE$RPDsm@`6^H`n>;&#%7j)85#d+FW9DnU;wyKl&jUt&KNRZZ} zos`NGr;}`L<0e1kF5VTlXv{5M@#U@A=ECL+@^fn`Ovrec-mlC1yKo4^vO|4(PLU?q zIQ9c)WmKRi#jqg#-1CTezEK#J_RpR@HC9vJBE3o=v9PXk6Q3S6$TEdb$2h(^r%YCh zLn5MtT@AqZIkf*3gTYJd=aPZED z`rUPfDw>kYHg=w_+qKc*L*o3^hZ)0((IXTtJ zaH)xu-7^K;lF~T((!bA0{ zL#Tb#1RW&CQJ@Z19{BRJURALHgdSt-TtR=VZXB$tz5;0XgPEJH^CfS@XSEuzN39-F zsT45t5}!P&77!N}K7|#wrEH7X^v@v*jlUYjm3$P^?BMTcIR6m4tVZ*%3+tZ$&RoJ0 zJR1~cWNMf+w8Ni1-HVD@8Fk7R>-LA|k>M7z8=X6Q*677`dWUbfHYD`QD<}*%nMdv6 z=H{-MiyKMk@QY#iP>iFuc*QQTt3T#=7m@-KVJEA zmK&gQwzjmc*w0?)mv=tzm$hBXdV?eKX@VmASO&i+r0Xf(T2hi2Hs}*s!upPZ_ru_| z#n#ygFWyTQCzL;1F0fxT%P7IlriRx4DUY7QTGxtu>PDM>Ci z*d}QIg{Ckqvm7$cl%Pit?KA*AZ@%<~_v4c@HxxvA1pNH`I->)0pYCmIO-M?rkCM5c z4$>_6Zrz0W@o0bh7nLQ|2gjs>W(w3f6X6$KwsK{|BlE&|z%c?{?Oi}DG_);mk#igf zmoZ#i04zNdZP~nKi|&xMrDa;D^pnU`dzQ;Tx+7EBO($y%e9GD`vHcmp$F>V|3j{;f z8_9VO{Tk=AYMhoR(Ux4&HJNW~x;~cIaaDFn?VmUG83q%=l^W}}E9C0Fu`TFiK2fR5 z=Vp~0Cd9iU^QnK8m%U}}iS*U)17;}noEt(wl)qnKpu_kcF@ zsaLA>*@?w*51*gx8&bL_I-+d+8ny3(H6n+2IfU~{v@aQ3S8%P&oddUa;;yM$ryUvduxP1V)*7y{xeW4%oFoKpePDw95uOUWQV4Y7Pd zwr|kB@avQ3aqoSY@y`+)Sz#1qIc}|it&2a*(6Za_ibgT#;?aU|Lujn$l-`7 z5S(ma@@)Iru>)4mP)Atzejdvn8TT#zR*f}63Pu`}DkyappDmF%sU10TeTOj;vn_qx zRBA0K51i#zR!vX~z{Bp!+kWtR-vba5^7X+D6*)K#Nx!1i>wvn`@4j+ensuH0f%4rebPzZSPy9ZJ zdol)y458m6l%Xsb9;4mEEcdiyriYWPh8llt9e1b2(UHj3Teohh8F&xMjbTt)0!uY}Utp>P5|Rlj&Gm*q_3@tR3;_2bt&hyKZs#xlN{)24}UmM@|)Uk%{!1Os1+CjW!^QLqyf@B z*oWp+s%2|weD*y-SO5XPlqZzXKKSQlgjr7$OHBKzo$Sdg;zIuDwy7MP_mN}A}#v8eoT`HgY z(pAK#oqRKUdS&kZ{KS|lzUsi=pXXvMi4PvrIPQ&FYu~-=H97rhblCnvYR*{sv1!;o!7|!hegH@nc zK6v~%iDa2NxmGX(y9IubAS^Qq{=5?j+IL_OS=FWeIq*{9En}Q&U!G%>WzqsybS7P(H}0 z$jRNAoSNc&xAbc1DkcFn!Q_`6eW#-3RXTTsH%PK_Ql3i^44PG zPo^V+b}ufMzt0jd7~}0VbxUukYppxGZ&zx3q^rWKFI!)@`}{v~?iyO~?JBcppTY;u zqy8mbOSCv7_t3j=E-?7?NZ(LsleM!!@5h*1M;3hkQniZ7k2-Y7EVw<>MI(qwl_pbF_)|x+@8R@F(Q|IjcOFJYkPce|$857#nWTUb`zL>Z!z{rzGLF%KHl8Ett;AX2<*Zv$6ckK z^i4e>uJ7v1VWEH3>z?CiSGKCUdYy@-GV2Mm*yBbGucu^YIzlS)S;u{$&bGJv{Neza ztACpB+~gLOen^yu{~jufiH(oH3u-wHNW$$%L7AOAZuE^PFlV-eMs?HqdrXd@VuDeXY+dX55- zIS+Iii*mGw+4HQ!YvsX+a3sq@4}okTI_wa*0uew$^P0i+%$3*dK|Fg?(E;_7*k4d} zT1hdx`}C@mRa`=tO5gB|GB;LPajJXG>cww&(~5)&(dznI>15^xmWoOLl2wQte5uXt z8{1z2I9FTcK3zH26seD+JaZ-n5;4r6Fc0Oa?bAJnDr@K!gkC+$TLf*1ZI`TSVu*${ za-&M)@e$Cbco{BIXzA-^JAnu__-zT$KoowDGJt9Wx&`eL`;^S0VG@Hx(&%%`?e8ks zP5pIOC!K+BV+>qPsbJl2=*GS5?W%RDf=ss94_of&l$<=)9VqiNpuOs=qeOkYjQ?@J z`}ybohn$ix>PX3fM1^`^&Rh$fW#8`H*fZH`5pqeq@9E8%js*6LOTa~%gLp3WXT)E{?&!$YN>nSA0cy+cBQ z73N7K7>hAqEWm5uvj>i66k&AMf88c)Z1izd$4~oF4J@Ejkz2{>tA<7{&?&LhmdO}1 z|6<5%?jpZ+2!81aO4Win{^;QxAD0H5^Uno=GBX(9iyAw)1QZW1^z_lRWZB{g z5_gk(s_vB>LI7!9W%*|qyX8)0@5n6%3Yu>Ix{eg!;7}dEzcRP<37nV<$jvF!UtY^D z^7o0i@ZSMdkx0YgyJv<5Uhg*V?6FTv6t+VQ_Qk3WEV=OiF2mxeAqspZHDdhrvF}{d zU`*aD@BjO6eZ1}@u$IOySg!xnWjZdW(fjojGpj&egg_0&%_@!gcz)r-M|cWI$6F_n z=UwRUlXMyHFki3a{=~d|JR?Izh<+=OTSh*qGRgVyZso)RSd-bca_&MJm=erdN8adr zUJr%m6jKbun7NtmOJzKiU0h9B6_idd0G3jfud<0FNTWjuI}7ih3DFn0zL=cG8n}k3(Z5Nle$W8<%-Pg2F1+j( zd8iO*4%wjJ3#uLpwaiS}+%(wR46duMUj>nucE}me@e9{d^B?8PNb*`_vDXF0rQcus z*euZGs(gp7`?R=xa2h%&G9u~uhJ=rA@>W?s)hJ`4$SE3gfQM`rsRT_Y;X`Sy`ZHWT z+CMjBv5kG?LUY!unLSAFsb1+!j`UDjSesB6oUb&#IXN-6AdO?dTY1BJxA{PKK&^Pj z@xF(HkQxlG;3m|GIO+LUxFJDM7iI)*&MTiHJFQq)z#;pD%^?Mrh^6>u7dnH*>dmT% z!G>eG0O2&$sI9egT=I!{=}-=t{0Q z5i0#pkX5wbOkfw(i^(YMupDsz*E)~c2(M$MXOY-A*_NQc+^ODFV9LgHLAz?KSS@?0 z?xFSlE$t-x(9w?kARAXnQof4lPZ3!yf#L%}QNkbRN-DL6p6H6z9>cCIzorejrSHtV zm1i{gFzZO_K&KGU4jg3Z0m1CT^WY;Y;z{*6zyxhR zenUmOh-Hs^)kQg`w@Zjp0QH|Z8(2SvGd#N*iRU|cMXtTo+P1U`cVj%|1CNfOShl{F zQGFy4nngIerF)hP>j-;+saq; zRNw1#-QC}nWAF4#D=tQ+hP0`r@TNqWI}yUlbE@Z`voTBUu3Zpb#k==}mDUP-uMd@& zx0{ks?8X(!ItVRd9S*=qj>ua;gJ#2yhYL{un^4 zU>bppztGbC6Q-w7^Ik%f01T};73De{-=0OP>(1S&kSt8fdl9bW?^X_EvZ2w8eC4Cm z2+!6w6ItIY!yR6jqK3#^3=VwP@7}$mEr)s&`krK%P3q4CL+U2GoR+iUp^nSzp8)tiB4s>jq&Nt>@^#r%!z>Vi8Vwt4o zj5b+0%i?`+Z?R9SJBsmt#m-j}^1g;*w;G1UcM1gU_V4}dq z1G=UY_B%*&uD!BwOlj9J02*>N<-LCQjeAP8GLWs*?^c{?MtbU6uox*29#4{>U0$@=4Xyf0ClSHi1r2 zlPw?5^aYS`x)kGIynliY#$@>TPwQtDmlmG8EdG+4!aAsJ>|Mj-tSLrwWk#CFPg3<&sikXqaDJ4 zsr)iiNIzdSjA3=7a6F76ludgKt(JuL{kIM|!9d?_cLT#mfKUY<`)SwW_k@ zVP7rfsG-+eW?I3w@IL8{C(EkEP4!R5-j-NQs`@XNt?b+%3!ZJOIuWH_|4-v;-O=um za^>l$26Q%nV?#w}hyk5bOtj`xv4n4=L{zoQ*#_!J?Q%0|pJVpB80OSN3C&BFDcgKY z+SA773AXtz9Z|i+u8AnN$`3PjF)CDae`qZn5yiHaa8g3nUn(f2VSG+5h}Rq5)X<~5 z2Urm@3@^gl#`d(U>SO#O_oJe!wTXpmbJ{7slg$pBT`$sjb7~CoMLoRTd$t*^n{?v` z=D3=7Cic>ysLUYH9<1Wq;%0wJx+5SusFwo_&mt&Qx;*ji8^Jx-@aishek#3qGzj-q9Dw?622A+|7b^454z6Bjtm^WtWIGH zq-Y>mcqt>B6Q)lT61ZL_>R8! z_M%uF8)t?O&_`xl1J9>hGL2y+rHM)XB}Wtp;=ajNF|M!bO$&!;ikXj_wxX1*Yk!4cyV1vi;Xqlzd{jzpBhw`koNKP6b$flzFGV&ii_0e&| zS|l111-jF7b)*~Yp+R7VJ}h36b!L=;Mi1pHnzc!$wMDv_+IXg^^xVeQrLlUunoMJX^Z86<${i01)O}8 z^z{9nTE}iT{8p8xqI53@d8@>y?WM~n<`LDX04XWDb;t-0koE{QQqxpNt%YvSJ)<^0 z|MiWZNNIVl6C`8ty5mpMwD>!~;3VLk<8cf;%tXctPX_K1+ zm3$`5Bl`Fy?a*Q4V%IWm4@TIltzF5Ji_$-{Egzk_f^h1B&g63CwUw`z{U^bPtpO=o zlyr3Ika9E$H`rxe*nb5UIHWJFBfm6zUqOK^Ig2xAQXR72+FVbnPk3#YrWah=jl%WtN#gt|C^bMO98+$pI5I|oXTOC z9xi@++sz=62*pc1856A>8Zn>0vE{(SxFrF-(*DnfNO;jf&voqBu?~F#fe_3Dayf&9 z-%$C(z=NCnIl4g4b-pc2CIPyi{U6PS{iQa2-01w;*Q@|U{MS>=CFxw1@Jo1gdv0UH zBGO=)Kx|@}8_#~qp z-1HS|o0_^neW5Xjsl4Xr>~DA$x;4!!7jgnmm*RE(;E*zwibTthjMC7}2l8$@SQ(*{ z(`CA4e%;iYCjV_{$yVKud8(Zb(|e{gZd6?6JgcEHABWQ!Esr|tH*b{2SJ5Knz3Mqg(_6J9c5NC`IXhH`F7 zK`A(qGd$WuFYn%`V`s>4=MNt51EMH`;xW%e&Sr6=&DL4_tOmwEC$nsCH88|^C$ZNEb5h^r%8T$8~(Su)yB3&=@hBOI` zSR@Hrzc1zzy^ZXisig|70b?Nkw43@tpId4&89XTZ;lkyEzZ7lAlS+?9?5)tH<#$0y z7ggKn|m>rS)E9XGO&z@igknOGDLMnJU*BccS&~jx;Qr zO4;=_CH{xoKf2n{Ibp07PW6--ayzYFua0hF5#(ktl;@4%*%B~kSqeLV=nz)*CTuR} zFWf*S(}RPdY#^i})Cf*HM^kg*=LECjGSTV;h%X(a}HnGGUSW_z*h$MlAK<9 zCL5Anpb^q8w245@*dEJCdbBbx$Vd#0HskXB=h9?|%guaff*D&&eXrCGrRQB>JGi9i zUD!Lj3vHu7txI+0%+O*9ROt@0Z=cjyXj7mn{yf}cI(T7m?5pV*mdkUlpY0xT*oO!Q zi}y^ig=^PIxeu>vx(TieNCP4wA~jcGV3VPr@6a#oovSG$M^(TE4oGfAQI_ zO?KI^a>AoACk(yqjnm>96s`&_F>`WVmz7uVhzK;e1S@!Ep2`M1|o{=bvGYFWK(^7UHZ z-i;BQ%mPQnoekz$zJmhdcH3~9mox+?jM@v#v=bXcWe0$sMO$^B3JT@c0DGJG)~KCR zq2xa`)hn;d{Ois{jCfCbxsPA2Sdv+rdY;e);vDTaR0r3ocoP;6k_UiW0l{^r%Pa`+ z`o>Wxw(b++V>GmL4E6%54qr>F{3_@hz+}Puj21DIE&+t-T2Kjbh${Ts=mhExd+nU2_k9& zq1E%`VGWrx_uS$?MBi%5&r;u0Ng-DM&tR2J>q02leA`y|xARQW{Po}}d5h-QpXCduB z38E#mAqO}B#N%N9gZ*TF#{O8ecE$G?u!U^eTg6q?mLeeUS}M3?^mR;Y$0^IVMx-+~ zg~j2Z@*yWx_b=eUw7pN7d?linO7BQ;9h@9Vm|>E_i@28Jw5y!TvK+UT?eAr|u81$y z+bljODqUG@x;eA?_Qs>LFS{xZrS>0t;jlu&_#vz#4fI8-kn)2lw&0ICb2~t|fc3JI z#=tEG;`jFyDNt@4%>Z}v^p+lobwp`YI}>L+z8GT5x%Z5scB5IJx+{fE{{DKdOYOSh z#X3%w&dqk&oic03e4;?-nSvR(L`qu9=-Ae!k<6!yA5HVHgAWfHBdtC+zRPzg4x$YZjIPJ(p(ECmAHXL2jR~$&@@*(5_4wcwQ?3xx*TY@q} z9bic&Rz5y;M5Grj_Iz~0RLckTSoT>qna933Uva-4){;m#zB9ASprx*qVk%;g40RKO zFDq_z;o=o3g8R=1rm60<#+K18(X_=-j>`fwK5c`riCrzOVxwaYEqPqF`{a`?$xqTF z11So%{&K8?1m+S&IP{nc%;mME!|86pj39w$arJ6cX)L!`B%S}AJvP8blIKURQUCQC z-0`*%y|;=E_nK>8xw49HfaBByv1}WW8CAduNjn#s!PZE!L{P!U7tpqM>s|x4x4zS| zG1PkVhhII_U)0Jo7-CFlGKVXT8MN>=$g#58?c>TjwX)Ya1*6FJe`s-INI{>vFQ|T{ zu@NI%=%m$%_+;051YQ~@+tzf3;@!P<+^p*cT&Pq#clV4ckMGRVE`ROq$(iu%U2Q92 zp6u`CRMEX_rpCVWH57DSqt3m07!6(Jwr7_zm=-1xZ8d{iW z*uY!BA6zzimMN*U_bxZ|RIRlbCzV+p9a?om>B&dEvS*$DqTyQ|2cMSta;m zGr_W;^A<7CXC2LsRDXp&DB-iMya9{KyT@JU;BId}H(QgN(tra1nB>C75s-2J{}eX5 z6$Oid8pY1ejt7LA4KTDE#DWn6Sb`qZu1~fhz7aE%QHbCtILdS9Qqt4ep@+}_eku!T zuiA^Q{T96@v<6af+c)`8Y$&-nMCw=p3~xNRTdFE{^Ae(Zc+XH%UGDycp z$=Eo76Dsk9q@)?BV+0=m@U*GeFp+9C@)u5oIGVgQQA!lAvK~cX=&XDihXO#}opIN( zaKjP(Nom|U%PVVMgMFDCeR(RRH|JCz-G6hBFdsi~YSG1biHN}~_6K-}64&ic$62Epy2A4sYJ^s`~raOPLS6H<0_ zE34R9)4|O3sI?C1arz6dh1ir#U6E>4Q4mf5$x_=o9t6U6x9C9sRESj7`Ie6BH^(4j zZWUge>Dk<-Af|msS>@2R*Tjk*$A&l_trsuce)^6&HHRrH^bDp8p;R%eh(tD5BO#XElj1y=1!t^WV-6T|>GKCbr7T!39u@vs1R2aCSA!@~a z^GNaRR@>_3(_j(+!|>eqb{y;NKJ?CE7zDAK_EJb_?7NTbsSK?DyIQkYuNUT&gCyySR0-&5Mi&+g=GNQWRXMbB;vEL*JZZ06@e$n%bM$*> zyp?!;Co`RHj4HU(xsmVs8SS56$x7V6U4ZgJ2k=cnRLnF?i4QbyuLVTpT~N>!DS?Ub zr$PgN{YTyzwcud=f66<#G(SX7&pj-LVIM&UT0_5d>gT&Zsx9(wI$DP8gXXlt~muO0zWUkIV>27 z6N`Wuaioz~lqx1T?jBw>uc5hE0w}#ePVj;Ig+Onfxk^XZ)d+j#d;)!O0Ypi=yjB2V zSmdu;-_)c5g|WGty?q|^MgD>y z@bueV&h8ZHbV3b`q=D38LIk~PE<32H$Mb^~^UwK5K3Vb`s-m_YT}SnMbKizm@j7Ua z6hJgDj;s$P*dyP@dN=w*HkyScc~bzcbsSyur8bi+Gy)A@gFoJ2xq_WExEd4Gf1)fsje=Iz8zmqJHIdLaNELpC4Aeq%Zy0)Bfdehjt09erHmet(~vN zE$Nlo)bp77pUz8#8u-dX!ujVFkimeQ!@idcNpXjmxv>xVJpF=ZRLKm)BVNuH1Xs zuyvKq!5YMw=ZO0rnu-dFi+#2P!iNH8p+5=zec{l9< zd5_B2Z|R@5I72T*>g;D`9BE!zhl?dts&aK4gFJ@E?TRf{p*K`dVwbXL8~3knc3o55 z3!9Y81c|UJJpr?FAi;b`=eXT6L%%a&lz~lOM3vQW-BMnQzE&7guN1T7b)w@{a_Qg= z3JV2?3RZnx^%$ZV$4MUzQ}Zw4wo`;}e@*=Upl!11S#QCI#{E$-kdpDwxO5d z*~us6%4b#oh!2oY-E;xmdS3$yTF@EZ3OKPDY8D3Z2E^n}yUm?LkIf(t9#q;t87i+) zB3QVVLy1P$hVXK9qsJz8EchM$a>bl0#k5zFXmcaBTVHPwBzZ9IkP0c8-L_v?e5){VrdYS5&n7FF^`_3^n+$s z*;)$S>-gXG46Kb@W|mY8c&lXUmRIIu{{^dmdgY;G=@|T~e5)z#gawr?B}p)HYYA9`6OfQTBaK8`5$BJV8sj zfHqlG$#D1DVz+5jAQJ{Mn<~92vifiS9<6>L9n)t|-p+fHRIrfIe_g+qz|j8rwP1PB z^NyF8QY~=cS@&NW0BEDE=l_$cuS7l;~-NoK6a__k58936(z{3G@Z|yQCB+PuQV-><4|A$6X z#|myy0Ho}J5!fOPG0>wSvN(rV;h0XXP4N6soi!8LGQd^r?YctpwLFyIxcCD#raE@- z|GTIJ8!xD9={GE~qi-cwApVqr;F|XOaxssWByn=!<%R38Z|Fw}nl4|m#u7{lHSZ>= z*tc3Ux)|?r7UHAJhFi7$eNUfi^s`G!c&O{*;RNaFdq6+H z6r~Fl0AkofOhfIE962|LAGG$aYa!@64ZT3h_;jhWxbHDuiD?(pe)=u_xwF<(F)vox z{KNF-*Kr!fg^li*;l?Su>AmrY43N}Z)7z^5IAOn5>rr?i?d8);yC+Y{yZvv5gXz0& z4q+YSfruyLr6EGYbV<>IX=Cc+u|>X^Lg~P2B$qNyLJsYzLFg5C07wGIt+6QV=@OUwx;|hH@7md zp*>(U7EdBVVg8&1Ge^$s>8x+qx#VLQT1P$l8eXEe7Z*z$*6mgLrc&DhQB5tHeX|g7 z0D*oyRS!OW0JR{qj|xl%aa-1WVd3(P)!=Y&o{hteM0nLr4n(x)%gq%SA%~SxxuCQ0 zPIanAX)eaZcKNMI_luOx$%(LAp@*Qfz7f=N;OPav4ho}N^X1@?g2qBHrqS`l|KuA?MKi za2r(YN%EpQ_K1BPGS?t~Iy{i|#O?fjdop2D08nGom8n?&h0y48j$yvv>I>rQrmWn{ z#ZO*&-PN3VGj!OMg4uG-)=}r^M3!fASLd7u_XO1T@-UE~14eHH4i=kg37Yh*&7Lv4 z6vz`sx_#rbFNxkC|F}qWFF(l2p>`_SM~!-{PD!1~^?}**bpB)+SE7xrl)Ts+w8MvJ z&DwKX(-D6RUa;gd0YXSkkg%8jO+2iBSf;Q%W8S*4+4HW?(!t_QK3`u-6~PZ8E(UTvbykix*N@;0$H) z&i#A@6-U=vQi75u-@NVK31aeg$h1g>gf`6xa(J|BF$ap-urgk;k4${ z8|l4r(}RoLDner)Z8u$3#i<`1G_jtEjjar5)Z^GwKGqQN7W*Q3rwYqh6uXC3wN(#5R^R&WiM1K2W zK9DtbUj-pOOqu@$^cxMm>)|E&=Q1L#n{rqL4;~SZ#A{H!0a6c0K_*xTSg&7Kf~_2} zMF94LgaI~*?&%^{ex74Lir|nMdrgl{85GKDu+>ZZydvi_#j9<7PhQ>L|7|KuozkaF zD%#FAx7jXS18besuGE%>+ZCM$_sgG}S=>1uXJ7eME9aa;TRsqG%KD2UN3Yv$72^%^ zKp60$=e{w981myQs{$`OEDU78ze9pOXskm-0yzpvwIC@KW!fb?vSjUcs zKMdN<`$lp$h>G+a1<*X(6LOnNg^vo?Q{`%Vip_%_Zbhc~cs+-Rx>$W@9tkN9;+fcE z3RCrJ6h&2}(ge#MT$=qv_wu6~v{eg-NdeJs!Ciz44o-qm#w z9C#7qKvFXe&#V&$86g218G5DaZ{NOEPb5QiEES0}3xfAs1`Nn9BBsX04o%Jy$6Bm~ zs8yJ1O$c3&DFjtt8t*AIo(;xvI&1cd6y^}ur0ba*DA$4H+)c2(@PKXBpTMD8q&QOjx(zzYRj~QYfFNP{Ng_(| z#_rIQoDJeyEhTNL0}+|VuHp6k$c5&$C^fB+xX#L~w}#)}=H^ zR0_eSnDc#FptF~1o%D>LrF)av=f3yO!z|CSqWDUk$#?s?pHnVNX(DKr{so7*8D5H1 z%b7FM7S5p(-3MEAbjpX7@Bl9SHu$h7!K#>E5OPT>?%wNG@VIGpSb|2I4eEO&^CioY zG4ONVYaCOUL3Ag6n+B) z^)za_5zLRq-59oHRZ0^X+#0)!26udb&7NkGNclbN{Fb(oo;TW~OWHbY_oh_qKl)PK zqpgE?g40ypU6P!lgRJCJwr<*afp(ugqJN7q(McYpkn782Ho|2G-iMZJ!KJp$7o6T7!}5pS5Awl#W_@-2_z+DF9~1@nnEUFDgc zU!BGlbYr~Q^J#nH;y6iqQUkULGkA&TYNI5o)4RHJB5!n}Y$yJ<8}hvR?)$K+iW|f) zX_w4d9}%0kRREe8gGHg??p^`&DBSy8VV*}O6r9lXv=hHS25@av!Og0=X-&m!Vs9DT zo!&JbDhp7I`%F8m3uj-jetz7C8)N)cU`Mf2DvfRTmcuJOw8w|uZOqQs_ax;1@aTo@ z^y>8luRh7Z*Z?sr;17bIr=?*nEGLNUtLjK;wRbc)CqNv;%BZ9Z(w~CU5Yw zAifXK+#%L@7;_4Iw1Q=4dJD)u6QR=8dC!fa3qHKIoO*MdVe&1@B-jO~ToxAN1gZ8Stf$BBPHV4z0~9LqV=JO-`I{wx<#^Of=L_iAnR)7ot#+d3)c+s9MDXgE9V zwh^Ev=>=JU=8p@1Yol|PNWf@s-KG9P6MOz1_L7I|hp8R_v*J%|l(HXHu&)CX1XZay zbP)L+UZwmM2CJyz`fS8YyeECsqXnibXtWGT_v6wC^)D7lK!+TM+LYGirWTWo% zxz>i@KN%Pw#W!yc79w=u1SflgPS~V=iQ7*P;78|cJOb56XZj`n!VEggjG0n=_;uu6 zfo&9KEf^ktY)=YHtl)?eWxppCEEFFv!yqtUUg-(}VgzIV_BB=l+pUjy z`I)G%FCEIofaZ?|kfWqZg(9iWcn#r;YFOj0;RU<aw}`-Iv}x!3!rbFUBCj9E#+gAr2c#*mp`n8hT*6P^5n5;!TgWhJqy zYp4B7RbjVTgukft_;$R_S$-M5%CaZRJIoqvS-0ALy?P-BfAK726fSol{)KKfNUsc+ z4OlgjvT{!4r2YAhRbSENb{n+;b5gzeky62o9r1~r_bOyg<_9`O)ktMc`5yJ3m{UTaJ70_xl(>e+-h5 zmcv1}g<ktUDdw88?>Kyx(-Vk0)#y%SR< zdv8FSMiDa1FJZoE)=jp3`ru>=y$DHv&9}WCHV_3qx5O`ZZvxu(rhmMm{?go!iwO)5 zyd~sxoA<6W7o4^7$?D&E0rrCWBY@-q^Zs8x9;+SpusDpSPD^V(VD<7HuXe7MzIx*) z++c(GyMH`uMzT>AzNe|4Bz%9ut8g;vM~6zSG}s%Zz_Ie<)A-C8>clOcMLkEbL$CB(_9(8wujNxUV}I=G#W zB(R=gBZCAe2=;;8djOSs9-lhq8GMVa9%DB1r+*i>yIY%SDPn3dC}372Pinh56ZQK^ zCjG>re%tF+1p1y>1&^(;qvq-(iQM$pEC@^Kwvs~|qZ-=bdQxMr^^W8(^_7eCCbWvC zWZXHv(+`ECbpAz5ltQPdlCDEty~STx9P94#nqxkqR6P{dcxLgwpS~+4^)E~e&4Rq# zG?inXMt15!Y$G){{hj@@98!K;@8*qnl|-@yWM^4_D;tm$GS7;L@QE) zp6^}?GpKa`eF2h%eFodqbkz+7$WPbs#m}tvln{s^QTPi-#nun>j_oQQGsViK`R?4o z)J3GZJ1wB0yR)Xl%Ajy0K60ef8fmP%GtWmCtWS6Qd8O~d32&~Mk z<()Lxx@!ao*x99^;m_YBQi;O!+Y?v#f2}=(pHK?Cm1KC^HkIxI?N(JyO=;EcHqcmg z4GkUOjl@(@X&@Tpg=XgcqYqnzZd9g^sV()7@gq-dVRth6uGNt46RnnH$OBoOmj zFRIxI&>vl!=gUWMZ99a)E(lJuFeX9$pu%)V*!|4~9wT(Jm2`dUv+dZEdW5?U)l!6H zjo$3U^@49vJdTyQzjh+zHxQxZDTz`nDJk3J&qT;yFGm7c=Lr2ZC=Vp!+~MmJMi2lyQ-FTvp(hO29N(=Qg#S|pAl?zd+mK>3Z+#~eOeJQ zwf*vB)5gXDwiAwmT&vTJBHJO1+z5_<%ot%~QvsStUS#7DqX{u;V2JV0gy?^sBc?xEy`6 zdUq12ZL#8)UHeSr9v_vvO?KZRZO|dUw2H=79%RA_4qEOF!Q)+K7s1S%a|9GS$;3ez-4nnkAxwDc1&#g(Fij-qe^?v844=Wl zl8p58LDo_j!IP&kU0EZ{v@QK^Wf~C4g&2y{a$@tD+ zd35&tAd^en55$3#%ar7QE~`MWxx?Ot7uAluRoHTws==nri~WKj!@m4$^_IuCy)c!@ z3Y0%+9+vQd8@(rLuk`51h^_zw;}`^+;2^W85kLviGT%&7S8Z2 z67J=na{{$MVmYXzg?AsO&V`5>@9Gsc4+RZF^Wv#n*V_B1^fKEz%SFxhjf;7+9h(*% zqL}U!F0G^OaQZX65-_a8_-yr)g{Q^3xOkWmHw9@Cwv!;P7yFwzWXpmw(fS>rfGw^u zh4$w3Nu4c2b>BDusJa!tY3&+X9p2Y%>Kr}^8(DnYplaQ-LHiFMckbsi;tbRa`QkcC zcbtzz!Ja)i2M+7GdXrp4{{>l!EpR&LKlpSUT1M>%ZdAL}{j^5tsEez>1)>@nG2SKw z(Pt2md5jAd*+GGZq{7BRV3`AfL zYgpQ(K()U&B&e8`>L)rGYLDDn^A$mp#fkUb))V@1PNnR_N^~Meg)+EkkIFnn9aR?8 zM{gm%Lx*DHmeqneykPP!AlMO4>$`B-pA-g#5OEK zIWX{)k;=f5>Rcj&wGQxg-3hH1tX|4>opv>N`zb@Nt3qq*ABR~Ti=*N&nM@|Ebx#Y1 z52$BoCxBH!28T=ofyOFTgKPN=KdPA$G@QzLrn$O)(&rooE0`%30^|2{W}svRG1Pls zkz_omR!VHXNNzUu-5ABCq37-RR|kfrTz(F(l90X12LI0f51kAoEFsf0HzB<;0WviF zyIfbN6A-!v(O&xBKvvW%^Q-tbED$%H>Vv4I!&gi1^x6~5IWN0hna}lLhEO4HA^j^<_MV%=Eo;U%zoQ zFKp#tBW0f2>Ek8aQ^x*)K=_`$JhOVrqk`zR9IOO{nqa`LD8V62#wel-1KGh9_YBxE zmgntn1dl6og_xK%fOZX}eAcIQxmHDOEpv9q0M*R8Yn339_BFhJUwq zFL56|4fz3Knb?Y9eaHq}#ws5?(p#Lc4Fe97!-ND#BAIrLw?zg~02RzR)9@Im zos$F}e?NFO z9q{XM@7~43BZrcbkuU%OR}h|pX06@sZAUe?|LyucK{;f6%LuCx;iz#1Qyn95bNDBLo{?A zI(dpN=h`irMmpaE#<&Y`US0@$6NX#}T7J7nY6^iMeKlJgt^$5j1w_q=vOlx3F;_vzZPRG&bTq?A||jop|Qkp|`zwO2jWcg+;!-PO`thuT4oi%}oU zL|#J(X9mfFIO~>2$1|@8n=W^0cQxxf#3`=ZT1Oi&N{U-IGft#o`FdExmC9>OQj)E= zmL@Kz`tH5295VAfJyaXD*XWrzd>CN^b3^#8=9VW?m}VRcHT};gp-Z-|MNQasxel6D zw_}*e45Q_G#YoDv$1;Z_Bss5>BMw3er3xouJeA zl(?hq7=WbFs!`IZZp2`NMsM_0at4A@o&D=?L!UT!oWIm?uS`@tlrJwe!;#43a@3tZ z*eo{?wJde^1`$i1HHAtUJPM-j`eRxMY1}x9MbeQj(s+>hd5B;6Lknx#^a9mof=VVo+ zTS3#)bA7yG;Ddpz|I~-bPzvjXV;I}s)`^YV0$B@{$dm`XDaTV^+xvU>d<)7lm4p{Q z9O46><24|RgFO_+1oC1xS5ie8!M{~gPK*lm`S5o@LX@Zjd;>adAtFr821)idS+FAk zTZL0$N!I*gv)nd%=ElmG;Hzoz!*6=?n>6~8)!cQMaFyS`PM7vPlmz1-G$Qj&G@)^Cn31P^nVe%D2dwMocg3 zKGu}gbk%Dm?5!-;rS%U7HrmPQ-O}GHG0O)xIY%A&;Zy&RXLlX?PxxNX+Z2gGdT&r* z3d)n&ONXhBf%1x^xuM!OCG~jR304yGj@pTh(+lsDlSBKL>Qk*zxzz$y_7-vwl`uc+u7(<({xdU4!r6mMlp(<#`2Lr z9?i~DFg*a9oOa_*?MNGu%zADE~>SF|P>SxUV8t+>jk9-ZgX#UQiviLr5h;~y}k@m6{4 zU`yQlulEy9?Rbb@!N)GuU0OAJns|?}9VS0z|H6t-6kEEcB)r&BTgNJ!RYC~kr9jvm z5#+h+;n)5gk84Ne8}$AK|>@4v5$C=if%D9Kp`1bXUt${s@M7$SY`h!H0S zbu6M01)mv``km*rIBev{!NSsvgo{AZHXEoXf#N~pwDn!4n340saSqhL$R0f1nGuE< zkstsPq9@HEZHaeU=EO&B*ywXo{eAjP+L{^Yz~uRy;@bb6{rzzUQ0MwqK}1OiMkNxGr)Wa2OGoQ z_&ubl0S#}9mT5#ncdjWL5L}SO_0kYwL~odUYldMx z2xQD*E;S+wg>b0{W8>rMrMn2So~FTtOzrBaz?N=rZzI7fP&!RjpSNv?2~{d_v8Pcp ze=~<$`Do=I5u#8UW?%SgTg&ayO}SgvKCJrcTJeYv=d;rIa`^U+mrynF6@AXBpV^Nj z!w1H+nZK;v=^ewg&*tkKbL*0wey(CkZ4YySC!;2I^vsAFAASrZ+02su4CJmtp z09YZ+o`$}2LQVei|7FlD?_?9|_KuU4EZ#9KJg}2e+?ycFQ`I@!6SqCAOq%m4ifrC3 zUw-bzIp+zzdOVM;~ykn_^jFF zGevy~YC%j3zWm{w9D@bDIJX~-*G|g`?cgoWv~=6gB{l4|Z02Qge3cHxFG^r@$CVJP zsLNuh;igN2xW##FPNT@ivk|`r&c2L{_oC>JTz4Uadh5c0#^1ANxjXBbe56oukLAUS z7fWE|$12RDGjEO-gZTNZHOP2gfl+mQcYQFp_s;9-1Sq<`HZ^UJtPltqgB93(7+Tua zV#4V5^BI#l1O_4A8d;;OvLVrkHj1D8lqr-KEq!qFf*o*ZnzPJm zx9U~~V%ea{ZexBPsbTh@Z6WRC{+=o?3#ry+5N!_WU+qjglQ0uA6tGYsKT$gaBerBF zW@dAc?IW@|0Cwag0)2c)!0;uHOLcg2A6d=ryBnrbt?)9AJh6emDhNwQWO_)N|9*b1 z+D>}v`NQN~bC}!<|3QR^D*6nmCCnXqs04%67)a4Fd~G2W*QNfM$&xu)9^H-aAEV|% z*SCOw{-|O|b>M{&@jKA$BUl2GABdTETR$5F zX}?p?h#8u}77xw&RYU@;HUbgFmmn8XN?8iwcgnv+Snbm zwsQK%;;{Xr};WRHG7SN`aK->6tfg-dIeCZ1%JmO$$_9AqouLefbk6 z^;+69HfxTKjcG-EzP=$@s3Kug&3G(W#x=l0{l53HpwXw&xZ-GTj|S?9mh#O@oV3-8 zk^S|XyBZ4|_`96^X%9JrRa9u?+vhG*qj|0s#Hg&WBt7M*0MK4LV|M^HB#` zPmZnMWrSP$StAY%o^@ zn)5EeI`$`0KwFNO0>KHLtRl}Zah^}p(T&2hnkyMYp^|RDKuyZcKbr^gy>a~P*K4bv zy;lDmd1>~j%T;c1yHysOJ>Ph8OHfd=NF{-41AGKy?Oq%TDaqTOf5sBC-nt--L*DyW zR9$BE75ubs9>3t6B7|C<++#a98&z=pwgF-fx$UZ$xoG$SP2pJ7u~#-9r{W%XaxJy;SsTnu5RD;g%h-C z)tj?Hj^~oS8c6#dHiqj3jk%iKjX$2$prn-$hYr%yj6IPc5B9e@TzF`ykuYg+#BLt< zk_Uup$n*{O=8Rib?w+G68mS2j8Itn%H* zWtsFF6~nO<6FZ+;XY#On6eOfRDsHk@cY<~e+QRj8{8gcSD34@IZ_|YDXk-Y&oBuml z6dl=Lf@pKzy;DuwdhX>#3EElxau;*RQ7>5^_LW%Sw?IMRBXw6y)woc{aohz^l>zrH zYpGT+`^>A$@?U}i&jZFuVJ0FREx<+4gZIEAU>3kWs{N2Mz!2e|Am+I~& zN=ryfcc-L+w3MV$-`vVM=id8#>yM9!d#`xceCHflKVO*1%n?9I+ysuM~l~o3` zetaA6+PtGn=G{ z>|qx;8^=}o3s>V`8)%FK2Cf2BFtKzw%cCFBbs}*>*}&1KfMmb!^r*?l+Sc!*1hRux z&!L?`b@pamJujeqfWdsuym0LB)UEH)_p-9TnW=lvpm$denEgU>B@KuUfNU^EIUecJ zi*&?x1;flgyZm^OA^a0kp@CM1&EX+t+`9&Ek97n$Ll@sCt?!Gwgi8~QLL)}eVkrR|cLJCI&5 zLN+Y|iLz}~d|=9h*#A$51+85|hS7fz?xPveTP6^q()rx)&<`;ZuGNX!y$( z%|_buLoO|PvKxgos&AukQ$SMSk%)S8C`7_%F=9tqW_i1=4%6}&h zJ(fKM7N*9tXGL_#Izp{lEp+i2|NBtXpwLkD3hU|j1&{1{cp@{A1T^@*+aXE7$#z_y zpqH(J-rIBmiLWxjTbRo`X`Mog9ApHs_{+NEBij}NhePZOhuWGFHV?ZyUv*$AEN2tT~uqishF4xs9Ar0Dh>Gt?QL+Z zd{2H8(pb>sfLBKh9g=$lPShD7;S?^Y2X}d-(iSEW#o8an0K z;v@E@mL1uzxj_2C8y!b3a&B&pwKvmM+O5LkM4qS;I+5wz%iH=NPS<};>`GhO&!W#( zwxH3KjU@Wr)QNmxr#fbOFg7bXd?a7M+cF78OS64v(x;x{dL1}hnM^fH7nA?*wm5P= zaaZ0c7kqS~KtB8gqX2c9pW*5(}pw|LJmQPNID8+_e@xqY9|Y2 zHnVz2K?_n+iJZiust}P0(-{~u2Yw466-EI8 zEg1M$fy^@?sYQ;Y1J4v!h~M0h9)htmf;%rxo_M<*n#1Vw|gId#Q?S?w^S@PUeWr$vQk9E+Yx(CLy+bOiVE zW})T5OWlg`()!RLmrGST(aJk2DW(;r=}JgL8l=J6BTMoTc|-BG?Gdw2Lb61^$jt<#+fF92nGRJ{H1hYIOU>ySxJcO*03_c@}zYw_#GYf5{I$%+<$qGV|#5|%o5!`nk2&o3{Tg7%MGc{TFOzwwMked(> z4pcKZnJh4Zr$AX)PzfEwu%fD&4msoD^eXq?FMPua>-OFta#%CWLp0l69*D#ydJyUf ztk!{yGUJZxC+jvZ?8uJBjlx!-b79X?f&c;txG#B9E5ig$>34x2^Nw!axydT!As8Q~ zp<5*|a?tVjqY;Ef_I{nXNM^9-;Wu=^LM%WKkJLhEHr-!4+FQ$k=`TUIQ4q7>NCs_a zpyH@hI2%r&`IHwITshuK4$zK!vqGp-;}UlhYt!<+#NUtg!r=*g)rku`qMDEzB30l| z1r}MSoR6k&tt^A2p#HAkjH_;mwN?ZkM`X_vYv;cpAtLb}t0>UU3aGda1jrUETQG+5*m8F*C=17nu9*-KYP868vXreJU}CKg3{*Au8w}cw zGF322LaB}P&_EuaRR9MA(x2^=8*-o*@~IEyDrX=S{AK`XA*sN5OYb;6(%`l|<&|u5 zbsW}UnF^!ONvsKa)p_Oppz?c-#o*D}C3bb=dzK%D{^l@63Q=ZqvmjOiL8vs8-$S4p zG=k}-U-1mrU1fd05wi3sK9t_3M&71BDu7rkkV^+xy*S0LpvR`M-OL$f9$!s z^fD5}EmgG~ELsnKuT?ctBAx{mD!0e|nXX09y0s(OL$ zyAK6ba^ObV9!{;Ea|wCEFgcmtBLgx8)Gx;k55;{%m>FWh63u=|YX|>3rFB(oY^^pJ zmVJo)>3)^#ugO}GZ`NH;@M`}3FJA!^L;!~}fiV+O+JodPtof9$us3aR#AT#2tHW53 z1;2f!TUBKyF0i(gd7Hv_avK%@@~%H%v!%0jU7d*;|EW4{JF0s!Tk}y>UF*9e8z?(m zpPmX-?fh`T_=)q8@w~S^3(dw7klB#VP6FEeCynAZQ);3|L4*Xu*pIZQZt*bpKw;JZ zIP1*ZnjUyMmOTZv*4y-NAKH&x#$aZMRLoZ#)9?GOcgAOv$9#j{ddzWMzu>HG#>4Nf z+JX(@#b)%d?1Ovv?om;TUB6s=AeDI!SNZ**RURtpBaMTcbpHYqTbdS$k6IDhWGrXj zlp2mOmt2szla0N~{E)_i@o5d(oDVwR{D()hX(KUY!7U~~+9Icowi>z{R~lS|9p3Ee5SD-S{?td|j<5;ZqSaC829%IJJ7OB^uD2jqtvchc5XZ z2hdl`WXS28-*RN~;Hdj@o3KxKi(#0VEaCd~zn_T9!y3dNo$Q~>bldTkhwO81saic> z?h1RG>5Go)7T(&{wj}9MT*h)zSZvLvj8BA%u;qNTxd@}G@Mfj&_%c7rieL%w|9h}f zk}*q3J?R~7qLj02+F`OMpl9p~jAI(W8nqyBt=NegrsdF$qQ**)%iX+p2d|tIL%gb; zhuP(7U0D3(Bs~^MuJgN@?rTDbMWBpdY&|U)OesnM9QmS(3cuP~VHiNc2}TxH<3$(J z89T|N{O=LIlDObDiyi4|`#c)c91{z3J8pQtt$lJi&3CMFp~+d$iU9wW3YR0h`6bWb z#EVIFa=yk==Lgo^eP6jJPi}$*hCkrHMgSt*Q&5o7(jw{X>hcHH3^W$q+@6bHl&j2^4$ zDuRdE3JoyHZW|b!22e(GWzy3`h7EYH46#e-t6lNwl=GG1W5mCR=6h%~Jz(bj`!wE} zSM`nm%+qXf2K8@C-VbD3L-n*SWJ!dy9eJSpMEsAhmKt^91F|U2ShqL~KRyn_=nO2W{G_?S*6L;{#R z>Ny1+mw$$7I($b>_K`WiG{Og?zH(Mptiu*OP+vHYRYFbtIwpqTv60dJ@L>P;n#->c zA*kdElrhmS6Vp{l2E%pH(0Ur1XHv+>Ctqxn9ozme;Z-|d#_j1H6G}`hEZ#{;U7G8%xhjFp-(_Cj)*Ywm>1oYFKlHP`gdjWI_KYBw&f*bLR(=jwqAW!K+ zaG}Z<$|hgAhuoP>EY6uncV*ff^$+pM6i+Ds`>+W3Fw7hL*U}ju!w@AWcROi(=^B^@ zdky6RSix}_O3={kq-{s>s&*FWORKpQ!)rkz(*~F@CzIqU(zO<~YFxbz^i|TSb&~<_X z0{p*cl-v1+xXCxAGp5tgYbT3UtPl-fm-8`DlJD%jE1{mO5OB|$R;WJ2Pq5a+-m2@% zLDAliqnnE7cSTB-uY^}s&}kzWc6SuG8o*461??HRNwM7B-M`MyUkZ`&hW?=;{2bwh z{2lONioMN+8<7`mePn{Rl*>QyB{ir%EVJ(HV#SF?CtMazlY~B>VEOj$J>xmme=g|YF1-^1LCj-; z5AT%^)Pae2T&|B5>Z2pLEk`lo9^N&}-NllkNzZOi#L1?d5<#R`-!V`UB7TcV(!Hcb7n` zLUHbj5B`4h^2M9`r#A#^3DMa-SV;68cBL5!EHE#-s{fC#qg@Nb>`M`)gj!Yv%p1Q4 zi%S`EMyr<>?O4#hd)vfJz!G!aW3nuI_`q(GqcJ4A^Lw3qe?^Z%#yks7U*oqf`TL15 zEc4HiyAP}PQ*|{m09ll2_YzA<5wz3ygH3wRIO%+KHV0{;YwmWA0?kmn1XMhbZeKt%|US zd7fhUTpvPbm3BIljpW3oi6xH@jT-Zeqb51u{sxlZZHEDAguAZ7NarE}qieWY_#r99 z-YXf@f$8w>p#E}BO^9kuRoBXDD*i;5a6Ljmg)ur_zznsj)K!)1W}xeF>85R>qUQhD zz}~=zx`4IVb|*{WS7?|wLOYhj*wnO(#LvH-X50l$BCVV=S1o2SgN8vXTdLynD+Bu& zjEa-=`iIQJ&KCWJ%gcN+T-ihpht!X+_tHO&{`Y(%Y?%dYC@3hf5wIn;fl|5HY=8y@ z4J{q!mMJmE$Q0tDTmVyWP7?(hnvXP$YOo;0l^e~yVSmK8nmRF-bLdLlKh?CO>r8F+ zUg;o?8N}#&ee%-gAXw7AQ{9`hR?}y5U&QytouUhAbvPZ-;pfk7nNW+28T$Qb^M#cLyt|4Bru%*JS?b|PEGTgb_s@bpqS=)RNg8;&E`d%Z98__a&NGf!f zzO!9n$Z})OzkFunQ{t0PA@dF6&UXescq_bXPkg7w>ulYQ=3Zur&z^OcI9P3kd24UO zZ@bZ3=h}C2_WyD;UR78jk>2`uW%KD4E=oNjK*`GESVwq?Q(uVm?X0_Mm8;Z~r|RhJ z*?!Z#*bb$@nBo@k&}wI~@qt-4MHM-uXzl;rjW>c?I9Uud=|@KEi&tZkK8oKjWV>s= zNZ@kur04T~D|U80nlir0J;_)|6T^u7ti{DXibNK7zE^Rt+@aR(xx@Nzu0!+pw5Trk zAIfRC;WH$(mFu5Ol)Y9lbY4`mu%<3BS?rAVXY~o0xbV@>T2lUiBCC;KLlvZq34oyS>41635gNUFUq^PbbS!E^4Ru< z(Mxu|n=f-73Uz4Oy6_U(yg-orA~L!6PsVXSM6UO(4Wyy3s0@G@=)6(6^gUQY$~c1C zlRN_cA(c^41n2IHl`nYEs9+l4$fb8o47r`VS^W%aTKlLyI*>IYDv4+@bUesM2OW$y z0vj|9wQdqVz1qftv+kYq@V^9I`61OBMhv)(G2&x%Ckj0L6EV9Qy8Dz#h-s$xEk$v% zbB?}RErnDz9W*|o6@HtO%1Rzo;Ya%++;sQa-zcZt6j7Eox4Mdfj*dP)KHg5^?ce^` z{5Y6fHL@osjVrfyw`UViBT`vck4ciW#J+%3zni3(s9@hSvbZJ^pLK#FG;vOf1PYly z2b>Xv-WO-5jEsz8Fi`nV=#BLV!uFyf9`5{zm&Vo;$Iw4pYQBNZVT$hkMm?vkxJ#Cc zo<@FYqaVZWuGHr*id9c_t!-)GdhhDV2L89|ild;80_zybEEPkbo;+`*qAAg2Cd0CN zD_b$bEE^h29TH7-Sw`dCkNI+aRdkBb3-LlU7ZuBU=#9Qej}@8e@c)z_GXD1|g(T!* z#Q+d!2(2y6I(Gr2T3olfwYD$7eUnRQY+#?LFCp9Nu`L#~e4T z-pIBV5Vaa^m!m6b#xni;zFOp|W>+bTdF?Xf+NdzcXqH=%05e+Mqd@LHiuh;1-N|TZ za3iD#4}EFYzPoyB7Yp#HPRQ$@89gkZ{g_IetVl|7UD@c{!j0DgwEG4F<+t$TOv!>x z@O7K#p5hp6_+RE9cl|HUkm5!0{23X;!@#-lEA(+iGGii`RI#M zn(JU=KAs*$-PNE9QLmEDi{8bK+B$Ti-zpHng&X zw%a)S%^9_lX)26c^`ZwxTa&ZYJWHd$NGbEfnvX2^k*9`ZkJ)V3aP@~ErJpV3 zS@5P4dEuCqvEVs6A(>wq@eeYgq@~9E?>kXqTvDJ^so`3i?6!`FOJcb84JWHi7|Q<< zo=4JGI!opZm%eDX8dpHY>~&T!YpNvV%y~ggP*}C|_sXXJ$!~}8+^pFN5zT6%AU?`Q zQyuJY^X51g5RHUqeD;3Ty34Ft&uw3iHR`q%{h{ZAE@7!5Wj#Z~xnp4JOi{PgwCFB8xI zRC7J2Fl%&DIGE=l!ayPH8sc+V_tfe{=4ldL4?bd8!RK}xy&WAii)nmraVIVIYdSlgd^f7`(e$^Qi(!N z*ia1v!H)SFKE82hJLJMm4P!Mp*&0j)%S6K}DzlV$vtJKpTzIh&*XGu=KKxp(SMScn zC)}|^SM^q!jc~qc#e@H<^b28934y4T!fVCCn13@xEz&6KU*KayQv=;qw6Y&RtV&^+ zrux{`S6(2o|}n% zD}vQF7l}JJiOC^6d{8iU({yQGOySOtyy4}c-a@dmtkC-QT29IfsgkJQN?c^o+KR}9)6`fJ4A;`?=8N8GE&9^ z+%oF9(V4CP;teX^!8a6MtZFj@H8%Cvq@#gDvGEbEF~N-BgPV6sc|=}VU`~;5);u-# zDaYvZsJK9&CfNP<7IE)>`Ce=Qj6xg9h!@y0JuW@F;-5Za@I#?dn{}Fx-~sk;R16=L zf0It*4FxCyZku91>5jkfRmO;(iV6WaHa|=b4;h1fmtRn@v7@6yiYdUKH7e#K(J=N# zKcR`nm{byB)=FIJc1(d@grp@0-{B`g8lH-xD)p?P`K>G|%=IbASna-j)IcD{t_Ir9 z$zjase?-TrITBW{yK{wHN4D?&1*rVOi0Wp&=50HOCvT&0U~WFNacRG^YbH!0Sd7ti z@WAhK6Xh~LNj-*~4=GeKCJ%kibF^SsSuJo{shnKTzKTIjm0=>klKfjr%H9S=+^dxj z--QzwZ7)4@=gE)~R0WqLkGCfq>>z@1Ztp>z|1Kd4jkX#!%}AbO_t(Mon!~1)@v~D= zx*dC0;p8*xOT5YcElt7_Mc=>NrMfIpyF~qK4?7VfNiQe`M}(e4$Z^jU3dK7EXKsur z$tjZOVanIoqk@Wy>pIv+MBKiJtDKR(4Y*?MucVBhPFNndY747xPq*^ zS_+Tr_Y?9_%4#p$TA{CRU);GE6KRvp_vAmYhkt|N*zxd}wvDXZ`FGq$pXH>Xv5*SR zm*TAOUO7Uqh#Y3menL%Km@B^dHbNg0p~P&Rf9D-2F5b~w7Hvzh-rQaHr|%2)QE9LV zx!>?PRV#k{=xT-$RJ=j6VRg;ffT;dCNWNA>f*0c7`T!81YEOcypN~lfiO0HSy|2wa ziTr&!UX`y>`6j}gPMQCOa#VXVO>?gSs;e?7(A3(4vmhW_5t!~j7k%{^ z#t(Y^qIwei>t6BS*U-v5hzblP^jBg{%F2$8364#B^meW0gJLOqpA}a~s~cXy)|Zob ztWNPpzUqY9O9t3gNJwj8hM^>?nC+m;jzw9>_N|)l*~XfiMyxI7^6-+v?u&u%oH$`^zqM)7=R?|sQZcco64k& z!4LVAd8%@%>MQRp6MnUPijMW6qFiw_D8rpfClPrgm_EA0SHrnpw;>ra)ZS|&nKSly z?F(vTalR>$*gGX7VPiWVLUy&MaXUw-UTpE#4;j4IB4}hw?g&#Y; zgxe&qfmAj?G8Ai-*Sx*;xdrPM4_W;Ee+~Zk|8M;H|Ca?09ioQ=qk6l-GoK`B^Y}$2 zg^1^8!tPa&1@+=Gn)tO_#Rp5o{noJq+p}T4t*1e=Z}?#{$e*L5#|k@3^!fM8y(yw$ zAm0a+PpCJE&n7tlJoB_}hS)H2g`FBI-L4>Dmh-uI;;m!Y^)3i)4;EpXla6SD3hp!& zz(xm;08YQ|HyyW-pDnWV_)>_n?&BDYC>SO0ImTh+-x`R zzj+PRm!{6n)zE{U?(SxwAqRtNH!Ty>mGbg(v)MXu+fE71Np{5mSfXD;L26OX$lgZ| zUrL$#!`N$NX5`xmt8lAn18@0DE79%7Dr(q6qA}QW>4qL&#<(-E>9)M9#t)IL0#WqwqolMnDp0utq8J`LUg(U0 z%o9gfS2uX(0JyxwyjH|u!;V0ZhM<8&gWq|J3Y+4NKab_80jLw5o_1bSo`1YVa@l7B z&Go5^L;ibiN`gwJBHNu9&V)4ep5Xd&g6LSIu}c0Qf<%6>5ozwvIiv78@!1N}h@~k# z{_U03zfuMpOQ(UtYySoGC-9zR-2Dsy*LiCw;$8c1+InO$ZDK+XzW2{gk6m! zZ-_}rHIs{W_oPYs-GXV2aLtcFOv!IMhXSOdh`2aEm@sui)bj?=p>6^h2(#eB`}d7N zF@VeV0@|4oj22^LViGquX95i^DpaMwMMisYXZ!WQ5g|Ul0kG2BJ37Rbl&&cz^1p=f zUoe>REASCsYv;EDQ|1;hUGR>Xu0;soMl+8ey}q})Xdl`cvukzdVEuP>Z*^j9$N(j4 z+eq*YyiA(N{{z;sP9LBswiHJEn(|{t zTq7j(1(CqLR)+g;RHt8S6+iYdK7O*k?^5wr*o_z9czZ!|Xga$*_~PIG)7)K>p2tog z-qU7SO;@q2F2|BR$Ap$<)#4I8O_k6&oQTav5gEMF!t74TNuk`K^Iz!AMnVUmh3VHh zd_Xp9Wr1w(@$&MP)fJSM;`?EfUql2Gb2cvDfbivh_zdy6H%SPq6`;n~nVFeFdW8dK zw}5h!F@~o+zhq$SBPZL5>R-hSd$cy8?P^_tj%&&o?g&xYAnC`(F`Drs@s4wtk@{xL ztlfE^l9U!8qA-!)jIdOTUz~Xg7)@2Y^=rE%25iFc=;+tTa8*}VS5#8+Dkp~lT<$`^ zM*ChtNJvP&_04JX>W^=YVDNi;Pfvcq-Me??7pGQ!124+$u+u-6v&eDBr?dA1=D>EvP#;J zr#F`6XRPpnrKRsK11;0Rr~d#a>h>xL6R;6Q`#^oVlX5n-M=E|zNsa#O^bCR7`(j{i z?+}b?IURqDuf9o1p@d-N1Fz`(RHfY^$cVNER11I-o`@f^zGxiWJLJW z@jLH^V;;8@%R0LpS|7UYMl+=xA5KMP&3%-Q4eVprViu`34*2^6w=?5MX&0xQcTV+0 zx>v5bp`-h)>Q|T#wWcOKkCI6Hj5Mo9h$$*!1Miu>vb?mkwAF1&#PNv7p*Gq74H{4p zGS&aVSEsmW^CmqW@$HBd(puaDj3r*!O6Yj1O0KoDOd5&m(3zyf{u?Ss7~s`$tv#(1 z>26s!jYB~NRIZW<{O^B{!3K{&*w}2YYFk)XgpAD~zwf}cHY39*zxM%kdTUGPz$NwG z9;_f&ayb>Fg)6&pTm7SEYaTYDSOcSzUdPYhw#wUz>yV#J(}Uk_MjWMo`G z=;`T=pbuUWgV2Jxh=PtjyZw9et)SF!soAk04WHAat zP3TkK)8Sy0VdXul`8HvAx#qVhc3G|b-c(5*%6F-CsnaZP#y+{`wwN!B&y?rWN3q^^ zdS83Apf>NJ%Tul?;&w3_;|r1c^^to<^S%O2PDTLoWiII7B4)|&Zki*PyyYZ?;;t(= zyhZX)N|?Bv^iB}$q8SlXyyc52`=J2=?@r6=>qWiD{8|py#&8ft9!HKDV*%;@79?h7(9N7gJGp}WbUUG>&!la>!+pX=y^{s1Tt!;MD4c%b~h1U!8;u|jNZn&@b6I%L@qot3MwjiWOE{Q2o%Z-2*^nJ9e>1s zp8)U_xj7-c%!jf`g0H+%dZbfkL4b6rAA12KW*FJoeO2By&xeV6ZtCB_<-x%A9Axtk zj9SY{8(b63>EB)uy~e*HS$bpd0ZOnNS<+&jPv508F_Ui&nLkd-T?^!^FD&yr30ECk zUeykJfx~vUF3|eykb?ZQ7&BeMhhe?%bDttkL%T%yfi zU{qc}BzyigN9MF@YHA3t%uG6k|U*_|#F#*)apOVwIpp6G3}dozhufODh0ouaSaZGB4sQ zc-12rcx1~Dy!^Y9kl4Jzcs667;_Gtm`rZ_sv~pf;r%iVwvZkMT&8ylUljl>-si!^r zv+mpQqwl}EF8CvUgzUwsTa|Gy6nDD!cnil~qzl=W@3c;RbNg1X9T!XxorPT3#sqr* zAx_gAU&xp_YPK$dIlVVXf%*CQRZ}>h3lR|!*{@F!Kodr@*0l_iY10bucw+u%RFK^w zZ;W`xz-M@&VVi9R>x~NtXK4Su*T|g8RC!kf1bKk1FuF(h=LvRC##Be@pwOXZ81 z!6d7g3Wsc{b&=bm-uP8S(T=sohhMLLwnpf^k_gay>JAlUf^lz&|HD0^@OBhkp1Szk zk2!SLPw0$(WxdlZWxFBJ5+GCX4-{=jXDMlNw7-t5YSISGiabwTzb`E{!Ld9A11-hy zrGL!SPBt@b_Rh>uz*;|q#E}#5n(@iWj8C74A&SLwn_mZ);X^%YQd}Z}QU~-xLS%@q8yoC)*N5aqKV9Emo4uV|S&DOaD@uQ?Ye{t5TGTA4 z{{4hs|*=nH*)|C*qEX1Jgh`PJ; zCzB3l=yB;pW`k!5VFIe{q5T>aesq?6>zx~ywpS&Qy)!n+65po(U+$u23evPX4Pt2QTdF?uATm;J+ zrf}W98PHDmvVYjZK!zMclD;eF2|br7kXWAE*?s%|9kwz*?H&zA{Qfm41_J$aw~G(3 zU!|&&4VOx5<+A3^@;%GHG4+k`H(AD>FFwsV ze(>CPG<-R8uB0et_-q?Dx&HU_-|1N@gaxa(xfPvk-^ijLF>_0SQQF-#op z>#GHHb?M8%rEe{Z5I)U5?lU;vd_t;~RkF8BZ~UEL|(qD!P(D z=b)9Q5=#*Fp3B@wa(FG`lLKyPi@-Cp5me@CG8@vh56rfJfMWiORpE%ap*FW zq^BoM*LyM6>dWa;d=y@~@R0w|$vZ^nYVME3G`3EehBCa|c3IzvhD-Tv_4>kRt8bjb zDXVdq-H0>=TxeXzG$Q*}hN~D7FS@Cn&B$dn<9Szo9ST9mT+H{s6B2iK-&4@eAcOG3UdZL8UUmGcOtZd~9Z2iBxv>59ZTyqgW=Ch2^2BarD4i@5W{+P_9Z zq9sg-Z*N)nLKz>Ilr#`{nR3`{H)+x*iubns1}7-00(ESrRy%`~V+nAC`bH0qeFix!pd0l*x6dm_W7^}kLr)N71EzSxs)Z(e*2ZO`HKSab$!*}FlqAf+KN2dUKVc7 zxKf%-TF98#9Z_Svx(S}^+K(z5#$&oKX_mNg>p4HW4MY|$dhQL?W71}mACht#TAAOI zB`eD>3+sr6GN7NR%=5U`VgqzHw;V%ouPf-4x&}7G;$K0B!Vmc2-28k*Vq!2PK*b3XQmQyW2uT@!hCVbFhul5N zE^%)A%lE}nV=13p{fXUXCwcl>&XAs@I-25Tp;MSo`TME%n4e!9H$0?TJp;ZcetcHG zQ@M8h{3)l%Bqx6T4OcXR37TE(#L%tqxPgg}lt6f){`j@ziVEHz6ccw^R7)clxvVqp zhVkaQxVbe=PFA;Cf>VVLWI^R=8MwGSa@c_K%GRBEuiQ5gD-zXqtDLitDG>K?X=bVs zHHlYv=fA4)w}mH+q(=+ZjvBD?J|#l7@sMO^i>xw*D_9|$WtV#$+4{RVuf^ToHykK0 z66)1=F(YKp+H8rgn>8@#XYDwgt$%VJr|oE=Rg3H;j~5#`r5whAt`8eIKKIg1Xj@DL z=PNPEhWOOISMlX8%Tc-|sDfoRouC&zR*+|{FN$yao+e5g(m;+CWH4@Dkd zd~d(F-Cbcxfd=%1GQnAPrXy}e-6MkO69&?!*$6L{w1Q3l^T)L(2V7VHuaFEx`8f7z zhYc_udky!4(1Pcz_`Z~VeO}^Q$@Q<+e@M33vnLLk)_6w_uk&NRz3uifA<%rW`${3e zpHyZk7H58?NKNUoT*vsY%#cIQ=Or!qi~OGJ8VIoiuMWl!te=Zn>J5%6acsZNIepS| zmR|gaporw=nZQ^~0Pz~OzBK{MQg6NngsrIhp1bY4Mf_HkT-L2` z!IFT z!hk?jwgfm^I#{w!ixMD7vg#DcK2D>hEjE7oUV4R50`z|u${&)SR*CTyKo5hU@e0%|qaPc@HDL~ivc=7>Lb|a~{X(c0y%XU@!J*K%o zsd|`~Y`lm83Cg{SVL|B1v0!(X#^01u{^Ef6{ARu|_32K7vy(0>;RCkD#BbH#HpWSh z*x4ImKVCIfl>PgN?2+3zkVW8`_=3GUEi*F)SSyN~EaZDM%)(G;WW12ey3V4W_ZeO@ z69svxVSY3BEcBr`y1NM{0$|o$tY&9_bt_^R_Ddk_r2CEuJ!4K@wmCljWTf$;fx|EP zrQYyc{*hX|v(DuO{MV*vXV@G|RO7ikmw$R9J~obuUBe%@`}Xjo0HyEW8nm4l3VVdX z)2AU|4na*vhXUiRn!zTDI*?|@edoSA45k7QUbD)c2|#!{Csmqs8S#0RXS`f^KQ73xniaAs}t$+Tg&vB3jLxqb9>(<(pl0wT>bBTb1o z8EXiLBvU-6jAMkLIj0LF;hO`tos1!(~cnTMu z6INO6)x1X@^n205vzpD!F@A34enGYQ;`z{^*T+eWVYn7?1rzg)Pe5(`EkVbxgHDyn=w79NH25P8&DiaUt

{@&+Hm*M1*MjFkKRoU+toOAfCf_mLKN7p`qCxr!A=Y4^B6 z#k)PKV`B)@j3B`e8CV0Ag68bDzyBLd4>kJE=hs6*vwr+S>v6qv(WkSs`Q1iOX~QFA zbA0M_##TjLgeJY<@x2`0pQyAJ2LG$uS6f63-9CyHPotPZftA5kaM?4?28Is5B`ljrB-{L{;vYPH?2Q^X~sf$HI`rk`C{#Ek<}R{gt*c+|NWf{h*1kS z12(Mz%}+l?(j*lSfq4YsA3UeC>Vzw!+HMQ*ToUw8-QARiIBHhP!z27pr1W zaUl({R8hsa96k zLfqBiz{LEGPO*B-o48&>#?A5f`6@q3$V*FKs@s>6DyjB-EDYVl4gNM2^>=srIj_}O zE8gp6$}I?>DWdtwJwtKk72M&kU}PWntAB;Ln0B>3viN?^k~mW?XHAs7Wsz(3i!mq8 zsv5OJ1unQ-tmlPYY|R=vN(pmdF1)^~l_8vT~lB&M@_Ya?=^;{%hwoI)HmR>U2+_P=EHD490sV&1@hbUN^F zvz3$x01as1NEYhV7MXMrF53(Gr<-l14XqS`0od;9$kny6w`XK-Qc{bHp6WSM##cV5 zLMK_wP5*(chZc&)JEfuYYxvITvrfw+uI~r7^|3{~Gi+HD{$a!CZYy=T>_AF&2edJ{ zxw&E=)bqCrUz}Y1U#cK*_Spg(=R(&FFgf_TxyfN_1$Rv)`WY#20h_`6goGetAfp05 z*0KYdN%i7-YKh(Zx*4XT3a0cKBIu*st`Rz|bPp2uSdTCRZVZZ|BQF;{Etjy5M8L|W zSc#mmIpbqy?{jYa`jY4?@24NDJ=yigdgJ@uY%kHn$TH+natC&;Q_wl5rKev6_rL~d z#H9|HVYUJ`5dv0WM&QnXRBwa=GY!sj4%&6mOq{q83$_?PJlcHQ7xeX{_Y-M%`k&Ep5knu_2l3=HIhR@b>go9G(NtPKUFMI4c*% z z#y29}9)shWhL7t*BYzLpGO(?#QN8Do^TBw4bBdL8Oc5(|P6AQRfbtfc3sAAKu`gp` z)viqe9tQ*<8hZK`ln!_ku$NLVco`h7r_%7Dnv4~8L@%FqUtYWU?omg3L&G@AkRLVI zC7|j&@H@d4V$rC~IA>JwV8H!@+^1CgY5pRRJYb+e?#L>dBCsw?bJr8UEzhc{^aTZk zQ}>)$uC2d370;)GM4#b4rEj}JZp`SGqdA8Y=AS2l+VcPN2o*9l8ePdoy0XT@hq=sb$a5r{$s)g# zgw^+ba7FsKW-}O4#`7b$7{riV(4vLCn+onD5~YJ~cCpnLt{)R+SF0V@8^Nh(yBY66 zDs)Ui_VzL?320M7j~yQ+kjrOg#Q~l_a1-#6iG&wFPkEJlr-I zxFzlCDmXYaqzq6m=v}^nKY+JPM&%J`o6?UKhpUM2)yvV!-VeIt(xM%*6uDe)?9 z0R--pzl1}43jytI>(3CS@4uCAH-Ke$vvF5^_1-mm0u_QF8Fa8ognx7NnUd- zyjW-%miwLU`CaP2rxxi0?RbDIz`||B)l*Im0~!FN>_C#jTe`t2d^d~xgp2v-pz*zj ze0aYdUMIzTlWR*a!N?OL1r`oXSDgNvKUC@SU}-f`;+W(axo*AeohyhY8v* zKW}gEDP{OVp17~#k-l=)!3&U+7|1v~$x71001HK3S@AyJI<71luKX*tM6+WKuF7F2 zMD-27@d`82-)nM#GamWu-Qc#4!@HdC4&t7b?7TNpIlru(CxgCnF#p!Ls{rE34gP0{ zbVkxz8GLDgppAe}0{9%nFdlIgkeqeV(_u1I#}W@?%!P_+C}U)mB`Lhyc#xYyR8?X9 zGD_ni=BoTbvazJkGCTgOi{9QTcR{9`cW0ONv5FIElG9B?HukLbme(^WX&P$`pMD4Z zy;M>*5E_EJRv?f}`%+_p`T_17vwPag_^^9JB|ajy;_rVM!ObQpmT&7_emMH;@?$l6 zjDj0JmPgM>qNY;-D4@fNxzAvse_W6G6nnv!T-FA8C`ebk-!JH(sD-#r%pC z$ck~%EPp|&sHfutGSHVFe6i&J-USPH1y8B~#=sHES5(BuGvnw82-oTQ^`4r=k5RJ~ zifh2oZ4@nZNxMaF;MauXkin!Qp`E5>{DOCHP zZl79R2AFiVXMXSVKijS*)u24b9JH9>dXZyYm-l|J;wa;PTUanrIy&+LD?dFw3hW#F zUO=pvina~#f5SQ!-Vmv+r;sUKCvK6UZrJ&`H}dyS|A)Ze$i|T}Zh+KqI}Ckvex>0< zYOyV!V2-ZN>li=NZ#lH{;X0896JFum?^YrHmU!=FpnSi6IO%yU8sym~ z;y2}n*gKWthUmLim+dd;+tS~u zepGIe?7Ai)k;*>@4<@|8fK@fkwFE5$1GPsc*NXcY<}&p=oz-%E$ct}f8j;C*tjs)w zaAI-ehL!M^{kY9x+oiQ{W@{$)q3RleBW;Ucqw+6>N7y7_Y5jdFe=x9XplyAAB_cT^ zk_^xMH`Si;6HPwZr#8C;&H8;Y01M3zR4!=N+-67fnC#B8?M1?ArzuIrCsTg?D}_)o;Ien7>Lb>E)={%Li3Je{@bq9Uhixz zr5-VU05!Ru4;Aqli>bCA_i1${-)Qn75%Y&mg~an%v^;ZuYFnFVSs|^d*(@9~$~6nT zR8M@J;)WF;$+uL#wkH|9#$&C{EOcjd2k+SOyCJtzEbr|LV)%RG1bd=2^=Wu>T-fpa z=Pm@w>&3{&!K1=B^2~ZnU!^Z3fZHS9b&%~ddYY$RTg^`nRO;jjAp`s&suC&SmZ(X3XNh+_S@+CB1XQB_^9A>jNu zHWVRCf{eN!?3c9rHpdDnLAdtM2ltDKUBk3k(7M9dB^S=1Bi$E!C4(B?pH!^}+9+j%qsxh}2DyTW+ z<{do$kFocT=dus~zdtf6BSiKH$=<8X?6S$oPRPoh5h^o62qjy}CbBo#D_i!+&I*xv zA7^!azrXu;|8x7R>!GX9`8m(`dAyJJalBs7<0hhmt@b>pwGiv&;Bw#D9rpr26ykn5 zc+y9Mc+Ml`gsRKy8n$f>*2s;`FD295zJ?c6jE?-+V*jqz+a%WwHxMP2jGP?BAk3@Y zp7Pop@(5!N0dHoG}|Q2z>wdEg~89fQ+8031p{V8 zbV~)1Fckj$%5HFb(#~b7;yqq)XYd!(8j@7i85h4g{r@5=F>DerHa$Sn4@>LJ2S6Zd zIYY|>A65Tz6Qse&nm6L!!spP@l?h|OjMBn*O{rh4J!0~_dO=W1;T!5}c$u%Ul_)2! z(Atk$sGXTllUMV4qCf%tHi}TdHlIQQF%1E2;TeS9l+V1zydOXF(n2lrOS?@DVw;}H z-sbaG`?o=AV5qk!XI{a^1jWFs*x2hj7?t=!zSysi;-WJ^1m^hWQ&m+}*B$og{32YY zsSQkTtpG`}-Gw2Ctu^Mm*_EAbY*Hg5;_@*n_G)LR6^sR5meK5x5x9gO= z{|v3?cT-WGMoRvWdq%{CqSLI7sr*+CSl&$d5pSw}peIW7>payX(U(10nXD}ZLCH;N zwzE)vZvHQNW@aG=5br~m|F%mPd;@7LMeD~rk=J|Gr~kw~(93^m9$bSBEv*iY1(V0E z^NEf;%Y@tzOB~6M3h#T%gS8){#>k@4|KGYIc2LEDk`knGu&dv(#%0CkRi^zwvu8VU z*|sL5(=g6420K!>Z?t?qtATC%_*EEWYJ$S@Ic!Z(7M}6dKV;zGYWbgQ5^+K}3h3fPDC?{;GC9gli%{jjZIyM!rO-&MGAu-~zcwI^{Y zxm7|Rr{r;7D#5Py}R+R z4lW=FXHb}DZX>Yy%g@~IG(I@^`|Rb**R3+Yx5(UcfwF7MK3?;40WUSZ)V%yFaXJsW z`z(>eEgXhGEk8zn_vq}Y-6_#_XR5_3{RHN>cQXpUB!8UxvT`u=M({oh#LIW$%zp!z z00n#&Bua@Q;iSaGGtde_8b%Wn6I&C`y^>j)@yxJylFwfjCCl=v70A5uh5DJ1K3c$- zla3Q$9cDg>r^2>AmQw4oc}IzMA_mW!p6|-s&ecV+%V83#C$#hrYPp4(Ztk;-PmH1? zhJqmBI=qs8C*z#I!T+}S!%4LYtIeqzG9f!EP=ypiGK!qGW!&e!77=N>^<=Vk7v!(7 zb*ee+kr_P4>LAUfLB6V0=9do!t)P0@wOeDl%L-3~vIL9B%QrkfuVo$>C$pX=31-N? zUFv++@a@YUf?0u`j#2twPF`e2;>hd`B`EQuAAGsSbu`fV@tbgj!U4Zzs=s34XU8Z~ z$c4D~a{ZD?JxjM^2IJO|Dn5Os4*2Y5@;)tb^Y0oHV9fh=EB8h;wCW!|=>DXmuB&A| z_irVdwX>(Co z_P`+GY;I3oes;aTVVO#O&~!gQ^%Jae1C{Cj$&1Xg*gvEKGX)(=DPLMzI;g>f(e8X{ z6`vWm_j7ieqO9)XmA99NlA#?syJ2asHn6&Tuvf49_#9!Mx79))`kBxov7DRFdF+M* z&N5wDo$Zy7-&(qgCD2s)2L^T|Kz{>O&*)M3eweIB9qnnBU!@e?1+0?638^2#Ok&IN!Cl5U zBjFw}dq}kZgdy|wu-wx3*lV+)iA_M$QG(Y0|6cgH`6JplZGf3|Z{7a+)9D%$6O>-4 zp~VHqLl01KE4_wvb>j=88TVx&-$!a}3OCSgt;7f2dVcz6Gu3YNkd0;Gl_u9bQiJj) zaiF^uvbeHwGNb~KQuxPM(@%5t(dBL@g?*3Z=VE82I^3X?jR(%U(J!!XB0p^j2vuT# z(w**jo#|;9p)4UN9%1$c32;pxAY~sHq0;Vi_tzIgLlVC0%c@5lhn}D5bN&3JVYU4w zlsy9#WOlU-I=f__e$0W5H8a0DVQe-Te_h_}ht8U3r|k%jo& zLx0z%s=7_V%rmzw_9-$UvQlkp*eun(w;gcnp%v%2^%MV|*Rj@Xo6u{{A)UqBrr*sH z%70OQ*gc`>ahu}3_OHvD2eTbZj;eP%r4BgP)GSwJ&{iDpP!o-)4Db*ZA$cz9IOSVJU`3MW<_P%VN5)d%hY7824wXC%!V=n zyNUkr7y=nV;6Oir{#?~l_Wc41V=_?>ZYT$#P;PE+Usqf*Gc!>rmz91@&`CZE4GryE zQ%w{OFlh>;1G79*L{kI#@5#~)0B${+DE~Bd)@65k))LJR{34rl+M$rkDgKRjuk6&`P1F&Ids!>_P1p~-Afu{%?Z5mU*Wh-KjN@ZQ zIae#cWTSY1pa|E{)KT9Ef{j`JQHQIhWzN?wT|dse>g|^D22jurU)6nK-A`uZtjd&& zG(zRofA3K&A$#`VsYK&IR@lX8b!ltdUQPk!|5G>re`wEuO+({zV8X8V`GQ^^3HT+J z;V=Vo)8H3GOh}=l@YAPf+Brxj#RGF2ihYQTL23)=e3{3OZ^C#i=hg~PoRW!o^25#h zTK(7=sl!5v8)}QqfZm#!nR&`*(KGXcDT&+DTT&7Ykq!X=6$2A<7WBN#tgKRZ?p#F1 zgJcd_16mv%2uW5?3&3SY!X}SDdgS(MkaCDM#$kmuX0Mg=YC(CfRPkW4f(v2Ax0bu)7o)PM1z-tO`f>*M=M3Vv&lWT^Okzh?ahqPF=mRI*_B zu1$h7;|^+S`YWMGy7)7r0zMPrzpIE>0Lm4{z9#}O$m9~r1EvJLii~^)K7x7RKQ|#w zb@(#_ENAdUpreql3Yw@$zMs_IfYQD3pmA8Q?ehyV?HyHwX+3ox@(EY(@puu~AVA3Y5rag*#J36rcNL_oA@lglxUOD{! zx!`$3@gn%8Bi^Ox!JiKplxg2T6P+j)OIWvsljay}uCPnKT0V7~$X0D#F7ywQS z!MT8=>nU@{0ZcCQq;_3NUgpmaVjo(7Wv<3d?6Hy&w;{X_Q#+r7Dm+6mPU+Q; zDKStDVY_$hG7 zxa*hqWMuZbrMZQU-o#LUjo z3OZv%EC#761)RpsyX6K)5wy;>&-8R%|T9weW@0l1t`26Uk1MkDQMfXHnN=lp>BYf2I zS4q!}dS#(jFq6G{=e4H%3BbDT2_lBHL`2RbN-{F8bj-vc>c5t?)FKlvu7rMZ%c*JR z7uuh;m-%hy&Tszws9OpuXr8sFj7A5is>c?#7|Re^Vu2RVMo#v*`+pNN>yR z69ds|8%K}hlk(pafE7zGsCxT20QK~7jvKVKPd%(WR6T^P9K>0Oc1G4)q5Wr)gJDE9 zmM-~|a0ocxi-G5B18$9^-gEzDZboVx`EpH`x-|KW;A4JX14E*OUozX7$Y%`Y*>4#w zFaFSe=0PZUBl9tENBmFa|?~ zdK|X*9a2jk&f!;^nFyuVNVh!#+I!Ad9tLYn-R^ypl-UjG)BaQ8&Y3iAD2S# zGFj)f7FEzDe)yN+!{_$`DaiIQ0m0Y8R!~jvp5Vz4;x2yw_o8Y%Y^3k#FR9ne z7mqs>>F-OOF8E;y@T}ubi0nqQsGqzepQ5JC59QCy6LY+1uS#(#2kXZ1Buf<;Dn&ad zOuQ=p(y!ZXto729&u72ozHPYY>+TMPU4~gt8Yebcm#w*C)ej?@M$R8Sd;)#w1_~if zzTGXtMNUdLw^d^Ac+-aT!>=>6_4uInS5X_XcOScDZO{HQP0@>&nPnaPn38}UV|I2H zg#yD~?!8avKw9;*pum1lTMK*zw^dZIP&aPgoP{Im(8z=e0mhGS#jP$4Xo#y7X8(Zj z3DNAQY7Cu5^f$~7mvU!hWC-+BdH?ujbLs;Qr+G0$c#3KUQdo)j3Q-RtH73h6yv-3Cxgda?@$ zTme%Nzh}hY;NYt9v`_bRK|xlk(~hUSe5+n^GN?L9IqCce>t=y_0vrHM67!~xRLA38 zzmL&a>6PbAPd-8aEW=B+yR;MkeH&O zknO$}7NbZ{O@;9Hg*f9D*aQfvsBiER6!V2(Og`n1YVjynN)7B;WVjRZ26nm7C`dgaX(w9b&45Wi-)k&2X(({-sQ zV#@L6n)~;UEtD}%MSajz@mHSZqlpih0<95bN!Yvi)|jeHg2OMA+Civ(DzM7^%ix|r zqRz=-i|cA2#3+DhoC+u>!@jAug_dTuY{!w0|ceb1o`1l-<8tVp7sLNQw0 z*AqACUNyFl>b5N3FWt|AvJzd$gD~1Mp-{VF&{P^2zJmfM z>MA|`IrwJLPfq`T+q;ZLxZ;8{hmrTm8Z-d6jI~%2AEkm&2RXzZt}>`2f{~r22$ha6 zn?XJE`_fB8E9)wU#5)f^~~*ERDi&S=zeR*MbHAw-~La{yoBzruLY(qUj{iK z;%l)HI5R#wBt`s7;71d5D5~^=b2$&<;})V|_0`eR3INp(Jt#ZD--D3HXgp@nmiIgn z^B0UO;>X6p;d$`o0%Vg~PDsWL_6a131p+A&o`6Fv4QC*q<866_hybVogfCZU)*j_~ zGG>>g%P^lV?P3gDBpPaCBW#`>-#KH%%5g5S_G`=)ib3gz90AVb&`{3yZ8pDh!P}fi zp#zUnmR6bi8B-X){_1ns!5}cUn3FQj#7XzgJQ&-cSOVYW1F6m4KEC@Q>=u}UuKyiG znl0kk36u4UG{d&P3G8_d!EUDu$TKj)YOVc7gj_}DofMp}E@o-6!V!xQ%u7<%)~tcJ zkl+u# zRk9O@Ad$kgc=}UBj-FIflKtl01h_` zp2UW}botr2IgNQ3HRRd^@+EEOzpT! z*L9`>g5~yw#Th_P=f)#|BhSUu{wfe?jaM%$kNjfk^7?*G_%J-*(Ma>>!!Bde=YDZM zmiz_D;E3P)_AAxHR12re$S$<(^g_M6SL^)P9)pwoU-Nz?n{L^$1~V}j^`-Rm^zs%V zMTIIcOhv;|k#le4sB6;@XfY34RM3wKBHG-=l&O)d>0BE}tZg4O@#beEV&~f!o3J`Y zljsgMD&*JJY2blg-mb`_Rws7+i5x|QDH*Ta6zIXRK1C%hpNUOSA1lAf_sJ947Iu^K z#`Urhaz*YHt=MH7+h>l51{gve_2mTX0zkftE|L05ov&B-t&i}sjI6Lq8 z^K*!L88%4E$;uL>>bcB0_>0rQ@|*6THsRygKtVnwPos{-yQGo2YV1(c+Ya74=22|8@FXl0N)QNow5tMycvI=Vm5Vjff}Lg zr$cE148Q5=73n-}WMvfZ)r|ZeMc_PQ{j0+K=Nogn0iSfP{r81 z(!up>R~8-sgRhxU%R4?UZMbL9;sX^gg70E7AO6YnOHBLuG{KC5Fh~&Cm8d9Het!PF zDmUBzX-~mWLCDty)HN!Jl2WB&vw1d%ta?BpmUZTz&fB|{kXw3}w4r84nm5CmEbAub z@?meUW0_9#|3F6S$}C>}4KuFpbmk&HH+R{8Dn!{N4D{ECeAqj?D4w5sUCGTCw#}CU ztwzsUTBs9`TNyod{_7@xW@eUT>g`CF^y(>@`@9^Y-7bmz;(s&mko=S~}VM>e%R zt;Uk4lu-1G#!}+EUA*=^$C*1+)yLkhlTADX-eXTM-Jej?vq|Sjp^11~S zFs*Mv{`9AIMzH5@CwlKP0KdqM^J{H@fesQ{rL#GFyE9ptk_R>m!7Sc!2m{jN#%Jgp z0m2CI+}pD)8@Ic}t>*ko%I1$-59~Hgmj@nVk%+ooNmgZRI19Upi64YQ5X8A++gue4 z|Gfvfcvs-FC#cZ9B>%F8zo*_s{wmVQ{Z)a`+{idTgL9~T#4Vp!iQlE2S&+_O*|-&3>;Lr zH%rjm!?zHjg^pQpZljHp{NGAycI2J1L6*@T3-#HC<~n(?#<$cMc~#`s-zWIC9B{c^MgN?4()56Z!t1d$ zvnp+Btw?|YLNEcOIht*cL;(&hEBGD{v0Z_Hs*iy{f_{~Y>N>vet2n&oHj zw7ZKq_`BysJhszUk2!7-FIkvBs#G!QK)VM)fdH>{BtNZ%_MV|>Vtwx1(RBf z`oD6~_S%Iw=8*l})TN{ygpkeX@#?*Hyc^+_$14Emdp&8)F#nZ{`A%p+O!SiB(i*D* z!OnVgdF3I;a);vA+@pJl>p%R|P(8tg#$5MeNu|BIZnO6S2WMT~8gxW%Zi@Np7vbb#!{A8)DKgcs#1@qw-8F)!uh4ERmD6 z`YpCJG6ls#UZ#&X&_c`bCF@F6=RYMS*AI8J zf4^CEC=r;_Deb^xE3{aE16HPp`o3Wt^536=6{c_r=>c_i0Z78p7XRWj$2XnKi5KbV z42W&=$7B^L07{xcS581n%Y{hPE9L*b!f)C0S9$&5_kwvGIecyFHag@-n;bAOgmB^&vXj-?IAbw5dB^Egk1uf9$&MC~j4JabPlq zfEav(Y@pciU+kf~@vfYvDxH*Ge2rQ7K-@nraJ?RV-B|E3=il|~MF*C$F{rbHL7oTB zGzdxM7!pw@dFYOK&0OXsW2veW%)DYCj%#R64ruo4z0ux>E$?2?7|Nw(=U#ZsU@ ztvK%2Wd9~cfj9>*)`;`1gb)CD2;gQ7=<7>wekA=pvhB6gBc_>rLlMy4JpjBDqUTKW zdcTbqpfDejJvXr$Uz>dnQzyJ(sBJYSP&9Ni$zz5kB5G3p~WFc0-UD?7$Vo>k-WI2Nt@${W=W5XAt zgNAPgm)Kl(e7pYZm$66d+W&SHf$0SRaWXSEZ-;(n@J3{cS|j6a&l}!<@=DZKk|h*s zPRIwNwccp?xFBzNuffW{OAm`uNWzTj^h$eG?BJ?1%N1;PFGo=Wff8dYqL~$}jK&AG zm_(77){A7p0MvZ|C!#`atCu}2taag=3@tEuF(NgEzv;h%EPF(A^}Aw|CM=jU0D}OO zHkkkJ{lH&-Ko5}l_Tu0)8WbdNsPzw=&l4?dnDf?WCra_hl+UOLh;L7q5g)Ry%=WYZ zysLHb?Afb$m&a@&Pk*A7O<7uD7D0 z9m9mE8H&r-zYTXziCh4;K}rs@zCftXQEsIN6kc0IAMhhLA*_c5FepVQ16u&OZR1c8w7vCbh128E zt&y`TEKKfysJlZfi;zKKe9r z;=+yZYg;al&X;t27*3uZDdD@mYh)1d;%`g1D+r`>5W7Xfb2M3@ewqS_n_8Ax7<6p+ z>XA;gppO}JqD66R9XSUnbfOm=aHvDxbKpdJyHQpjss{KOx2}GAZD=g1V}!f_g-*Mf zaYrB1q-54?cHD^eBiYSTABTz{>Gc=Et5O6QF8FVP!5y~U`YFy4hRw(= z#b_-u&>g*b@lU+=mP@ya*bnOtJ~McK)1Zd{g~P4u+L$wU=6d`V`qf=-p?PSo;9?{eI50j$!e%e^Qymg z?72+K%&ZtE@;5<>)a6@~UU>oF1jmbrX0t_R`1eli`(}jNZI);d0jf(h)Rmj4oU!XOVnodAGw8*$$jP)V&)mV4yi3|ILajT)a(O znp~l|xz9kK9j4-AiZq9JZFk-yLU81#D_-|UddA3FBn%mXnmM=t-%-Nc-HQRwPiWwd zwJ>^c_wRG@WiA+tqH0ud0yMm*lSk}FA06Pm^Hv=Jz5<=);nKc$ua~RoT0s!^Ot-z- zJ#7>2U%dyzEkyO?(@=x>U<7MQ{TjUJ) z@a^EfbK)tLa28|obvLKERc;1g2oOZAG;@=hf zouLyu2HLMmZ+VfnEgCBQ>?`n(8-Zwqe58L+5TaYR9L{~4m5zM+-OSvXiTpYk>B(F&tA2qh3x1y z;dL4(USVM(PYk4H*Dqt+$6fM0sT6xOcxjLP?VpFyc1>wYt-ZTjw$A6oVWHy+*g1em zh!i8B*d16^CGwOKFAf@{_YrC#TrXJ_{z9F7}%U7#_E3{<%GB&e2I zx2B&A-k|_Kxd$`){dsjphP9aI#||35CF%QuABv5$w~BPgxz0Yf zrzE}gg(bw5?CA<+lqq9R6X-e%%HX3It2RkyjzvK)BGS7ULMO$_+H25WswY`QB zPP@=V)7VLS6;B#KR5oNg7+z*eI-FYh?ScI|tG@Q1EyOxr$&>j68o*d64`}snjww}4 z!U(~wxlk@C;^x&};9|sfGn&2X^Ze_Em(tdD z#Qi^!TN3uUni(vqeiybmlUFvd6$aGQ;sf6d>RUKne15K*>fG$fNCmr)P6?bV1Eo@P zcf|gzu1qUF*ts}3kdhYgRoBIg$3G}j!hDFUVq!ID@GeDWeLKQv>i9&?ic_H2qVemA z-$OAPtIHNysQTAviJt(hOrHk9M$dJNs>KCg&VrT{c>^aUK-09?(_pzlg?nGhwMI(+ z`R8MtvUv6iq0-;1tZ_aPkz(fzbgwDAMG;v=611j4Z z5Anb3_Mvzv=s*Ruur$`rMnfMiH4}bXA(CAKb0d=YzLh_F^-`cji}m@{nE`SJo|!y* zH3Pfy$!|HGW%o}ewjKD3ILN^JTckY>_GTs+3P3T4)0q!yOp2_>zIm4IVN#x_n3|d~ zgTIsX49P*^Psg(KT5@t~OxZ~=w50ZCu2#J8bo8H4K|u?`UQtoeS5Z+=vONb~qsf61 zAPFFoFractfoK}v#ygLn%2Bf38qm*hf_E#`!~E^+u<^=c*#nOOlWGY%C3w>YiGNqR z39h)JW-P~X!%Jr`{JaEWL|HMXkjr)gy6{kANyECHD}D{FJgOa0hEiR50^y5J@<=RK z3lWKfO4sTV+c!4MqSWJ38ILIbU0mA@xHw_AAHlC*6C=VyzK31qA8khR^v|P~!AXXW zfI)APjc??5=Dx_&KzrJH-mN8>=|Let-~A3NiAZ@qo1$9w+~Zit2?Yd*Jh@^;&Y^D6 zG0)7Bq3~N#E_S~iTn%$_ZaqA0Ud2*eHSUhBj{5FdVeeyg`KMwuLClk!_B_qZ*_FYb znMWk{acYqFIgN88Z)VzcDFWF~8%$Ee{SX;Kk&YCH-sgHy63niy>gB9K8U1U9A}Oc> zAGAj??z?7+4JE%jyU#zHF0IM+ugv}A?ikfFAaf4WO)|R_a~#8CLWL@8Kc%2BqH z2EW%Hk!;pca8gi~VO23j@;sSjRQ$gS-q&q2fjD`Uo0|YIn#-e~7_qybb4D9ChMI<9O{F^Lu|@>h#DYT0d69ro`T5v+?RfxSu1G z%CYcL6>5AF7!}L1nqT`rkdPp#woA9mMAbvj0m1QtP5GX_zG=s67DW3BlcLWFl~GDG z$3Qm_H0Y)<&gUM0^}sKh3V<~j-3?pa;A_YB3xb+RS=pWUhE zl(*7XiLuT7CZgjn)o%PHSqArV=w`PZ{17}0#6}%MSJh)R3#=4+o+BsV!tu|`<3MHy z0~YU_%*7QI5X>nKk=0q<%O1;DsmNl<*SOQu(^_9vC)++&>aP@b4Yd+Rwry%p7T#)>w3($f9K3X z$#3-j{OZTc?x9+iv*s0ycIdXp?|7_6%sS2IUTDH*Z2^2AmuI z{{B~CARh?aoMf-jr(rpqd^z;8{(!n=ZbXrvd~f=@b{66uc&E~$P*-fJXh)P+{!}d@)JtJaXY)(lo?V;aEFkLCQb2Os-DNfap*|rn*bA@l8eYTz=;jnGcOdqPlp=^KF4QF*mMe-1fkid`@>z!}E$qt5v znDf3;40;<-e>#JjJ?ya%_}p7Y42m@OwtrZ=UTI@5*Yp-t95cp?n}~XU7{ZcDptE-ze+v+#zt5{#irA*RG6wskWQdWw0l+%~? z$x8Gr_|F-U_X@&DXfp!>*2pM9yJ#te&Y4GN%7-FXnV=n;9c|C+4yQ7~GK7Xn8Rt{Uiju=~Z2w}1-?zV0# z?rL~x5?tCrd0<+F^9A5Lxz;ivg*bK7X2i?}pth7}QF>N~jGEsq>c=Xql0RmsUAV{X z!`Weyj(lnW6p}#l3b-Y?Ba)`uNeSaMn zyOUS_UhIodt15f*XDIj9;UQ%R>zB41F9rmeUVV*yIuQA1PxNJjdr*9T;rm6e&HJz4 zH&YDmJlGegXj%@lnq3WczmD6mCSlK)`SCmP{Zrbi?e<-m563{C7MP;OJi9P>$w+1W zYxmiwRZnv*mi`@6``m}WZL}1vjfmm%SD&a7(idSDTm=?h&-MI*{G8XrkXD?tU(&ZI z_Nel2{MJ+D*=zl0kNNNvDR}cz_B8l`pY(~KX0|fz1Rk8Y=VqEQ{0>9f27oO=2IB?W zh>*mqMZr8Mc4P8uf6Y6|nsQOfEFqnu?{ah;5)@XC`#`V`ZveOluPr4=I`xpxtmaiF z#whNwlF*KPtt%y1L+P`9gq)!5jj4`LL;uNLG?eP@5zWcC7>6@*G1k_I)|b9`_Oj>& za5sQ%sg?R!ZT4lL#w%rY1xeA}?~6|7US79n)c5T+tRz1o?Rp*vhXX=^I~0`epKZj+ z*5mQ>^MUu8F!yEES=TQAVS8KMg_?U>OJ1?si?a%gXa?;tIMV&oe#|4v#T6=ylBntQ z<it<{S+%GQ{Sx%vA0 zn=tJod{Jko=2enn&?n!~*=#ubAa)IMPI_LU=U3||Q?E==>!qeoFRyyW#$014xJC|c*&G{eNlev`rLa(tZPo5l9@IGZoQ$ojVIK)W=QFWi59H)% zHPP8+Lh6=~7lizMX3wQAF1ejVdOs|d6x*B=7u2-lih#)d%7c~`fC@oUqO*nsDXQY!ACo;(wEvqaM39|uFArr#Wce%9b#3Zl)9eC?_ zbI8>uMl4>Wtj@NxkP?qne*7W4Is!CarsAy`yEPsQskadj_aCdcb>rU$H+NZ7ZA|UO zXxlt}=uPl7SJhRSSe7`} ze8RHu*i}#w*~oXe{(FxVUcK_B0#zg4>LtN$JANGldjZ|*#PKFQTjM3KDDykL5zT?o z_E&RHIxXh~ULm)IKI#f$0yWc47{^0lpb-F$?y>f{jv^wHwJD7jv&9Up;=0mPJ*)CP z9VZpEDgB!Bw9ac>(9<{%kE}iaCb*>-zF1hg8WxPuOyvE{R$wux&Ds?(6!5hrHdTJpC&QtCjL~Yc1!L&AxHt_wT`3I2Ew0?;GbmK`o6l2VNZBGQABCA>*U7h*$ zmWwqGtPN0(7`WCHW#d9I9!ozkV^KCeg+H%npO}t5XPLDqB!AAJKX4UVA(&iCGj-ZU z6ZiV&<|(kDyL92X+R{M7)`jYUiN(6Xl9xtPXq^x4c(3ph)C5$BPvH-`A5V2eI>Mg7VXd6>LP_YP@ASdzkNG^W?^<{lXQ&Uv2tCewl%K~{=O5;9Pz0798{+0 zwR_=>rXd%g#5gvk_7q~@<>-OMW9^A*O{TP*4?FKqQ%7MFSJz31`inj5Ilbl;-y9r2 z@4-mh4~z+s7Z$1*Z)4hi@v7QP!lx|x(6j%Z(qda_{micqa8J{+u=yYW z#MAX8K}T3E^wz6?1Jw74Qd*1v?6lM2kkF25e$cZ$OVptxkA7S5F=+P-s4HM+IVoSl zkaNYCs!P%Si2-Ln>#Im$=)qBOGrF+-xUe#RWbKt9l`)(-R*h*#aOSUu912gDWs2g& z_3zJV92)smR!KjVjBXxhBu-MXP0`GR+XyICw!mEYP7Y0QD32HvwrnS-|= zOnNLb``7FwLdFs2JLd0=^RQuRY**iR9cGYhx>2_-d}jZCB`)=CF;=3UV5>yno*Yb1 zy50++B`jGgRfezi7QZvDOq_Ov=17Uw!fmc#Sv@<=z3kY*YUb~8ABH=n*nH|PauRFM4>Pp>f`5DBSi&#f zbS-Il>$sPfF{-sjyb76+S9VsqM~V^^dw*WsW8$;EZ_R$I-II#-3&3(G>&u`vp#DMN zD@H2PV;~n}6v373LpFcp#&30<7Y|AB=ZeSqV>G`{S>_|T!&%GOheg>`1AR#9KbJUV zi$^a!{B!$$4VQ?#JlO;V!ylgif2W|I2%syit{ekDieD3ZUKLaz7g+VrL&&Iur&y0f zQkd|rc4o4F@#QMX%UVeeE3&zKplE{bYOAp-UM4GL?gFf z!}+j_5P+Xhdjz<+Mt68M>_y%&3J777V}1fXy=$$7t~px1k%WKF2tN5>;`$TZGCXHI zS?7=UGqnW&C6+^Yf3}~}M`GwE>vMh~)#IhJU)Nsw{=PaP684Jg&D*=I#0SUiePp7W zBCl;S^nGPHJ%+F8`(~Jt-r@>bCYdq)s)Y9Rv>^|ou9S+pmERK70V{#gQLi6fJO5_X2cCI$uAgOX)H)U@vk3e#F*syRB4pY zA?G=T>33ZbPDUXe#gLDx4>A-()d`1l)}=9iwNlRt2Mz>CP8_X;5~Y5gQo4pOudkZa z3xEWfLDb)ub@8(3Z7_G&mD8pf;Y+5zVu>+FI9H=Cyd+Hd@5u)1roE{^N}H-$6*zg= zDQg;9)+&KW*B{isGd6{5LuPzQY16nRRJ@d2N8f@B8ZZ7Cgg?zkgA`yr#XNghgS!ff z#i(Lk$T(7pvNv-0ODcJEcJ}doZE!h?f6}UB>BMtSwj63as{KqtW&7>UoMzgB1li}T zbWI*!F0wZcEb?LF3(sA=+qQ7vHb)>T=oK)VRVSWw?!5jQD|OTB(?qrD0n7p4W1Kbo zq{vvIpKh(ihO~X}upw*DEOjj1f97mR%U>|AqxXTt`kI9t>2%z}RE_S&dqWwiL#<3% z^??1iF>(IERVKmZlpl&G?tZ)AW0&`yrD?Siv#FJl0Q>P!54r^%hM-`%)N*!*M6>T-=77td*%(dH z90;@(6{Hrr+Y|ekeQ$F7zS)v}#{|OfYD;*7SPCh6^K@0wFUI())t%Wqq=+Y@CNzoxTX;?+soWu4YO*JC?D~9q!&??oxPG(m#2d zt6vQkK;0ebuU^CWZ90?5axxcY!nAzpe$2;Zo2?eqSQMb4Dh2E_M;7nfgq4OJ7w))V za83l9O2KQNi4k6EXSq5~t?XT;*y&GjyStW=Q8&LeQRYdD`89JxKLtaz#$g4f0&esX zqSNL*e_z+wsv8F@*K~j46th-Cl1xvp*x(Px<{=NaB)t2ZLl-Ju^zD;kKO5ZgotgB> z{MKTAC*&Md&mD>$VzWytkuQX$yd$qrNuhb+osa#t&(W0Qfas;Q}>~%9< z@`;$e+5FrH?Y;AK?OYd8Gm9G`{x5Bs_s#FrJRRTvfT!3|eF;r{_T@6y9)tl}Ed+_(U;gc*GZURb%Yn#Ue|nksD#TkTm2yR-HJp2}Q#ou4*wsCu{sg}9QcJELA36#gi2 z&qbbcHfmPeWVi^kH+dxt6pvkH7z)CLJNcZ5TsFI-oY33{IO&XI3^V2{k{pPQH&@Y( z;~qi?eO_R*U1t}*M;F(g<1!8ZyUao}n;QHpgb9>oK@%-#tIv-HpKw8%{Xb zvx}AywGsTy@cw?S(u!6Xr-e`e6WnWr64q&@`RZEVn_H*KIX&z;xu&>^IdP{Z_+a+} zM3syBY&G<*X!IK!@?R!Tggf)DAW`2xLQ%~zZNaFqxs~yP+i-`%Hz6ede$4adrb^nG zqmR_fM>?tR(_GEcxx|~QAy*pYM4NS&O4V4>fmy$yPL6B~ zRCXB&12)dCnK^N0+pe}P*(9*CvIYOFtY!zBse$`a&AwauNO;gdB5yyz?%NtE`_%#= zpNYT|XXEEJ$_vUTfJcxi@hfR0*%Q@EPm;H>`qr3O!?YTjyzsbmGWfd}$mfh{0cUyg2BdDy^=@-MX(Sem~4gXpi@1M7w zJuokai)s~vZpqHS=jUg(^Rokw-@lbJuP`G2N-m;e*s!wZ?h@-;w@*e>@I4F81;N3e ze|wd-K9!%6Hqg5hnlNC3i?A%60?neZV+$yML1~}G23x?t}tq9 zyj5c%WecF1R(W#SdolM`1NvQG<+E)r{~qtiyKs&rRF5hAP=c<-;kM;=iCv}W_Pz~^N@XGP;mfo2NPyENW;zor zEOLLwx_?UBvwBQrW*~t%QqTF2>g|H3>wVH6H?D;9^;O>W^)5SNR*{+dG9+@Rt*bjI zx`snoFfU@eMtbX>0}=E0K{5x}I~O z{~5b`wwoY&?7cy8D(;T=%qQvI&e#Qv!EOGgU}#lle#qRM`k;3tmiZ++Htor^hcc@w zw6P>c%~~8L!kwuj}NhhngGv?*4iiS%N{R2IDMJnyjoRrq_0yi`mg zmW>4$^ID0&aqmS3>(Nq;eRDf5X#8M~U!tp1)5_k9E_UL!+m@qB5DFMT)>mZnD|qOT z;uP0J^kv7M|A-%|h+nXn9s5D=PVCYCBNHuXibnp6F0C!?=pBa*ZI!7<%5{Tm=hZ(R z;lz|Czgah)RT)*dceA{)M$7n~NF|-nZ!8VYb@`tGHI|1HPdoxdwXBZ#${>Kp3BOHq z+*7&S$SIR*zxF|kpA}+8+Nx=Gwrsp0yV$vH^5%$~57`Un>^>TOwc#@ux{Tr8iuJ1V zN{k>D{7v4M7h|v$R8(3(0zd#$8KAesASg(}8iNg831k{)OBiJkxbb7k!$60R_yS-? zkDQAOKQT5YXim0&6!$&=XDgW3Sw8tbc_NB9%n{_akln;n7;Oi7m=})oEhhc%@4#J& z`TZJnZWv{YVtYiavoUdMEN!>HB%(_+jQAUlV{PW47l24q$m~CWSA&^4QQLhykVH%Xt4nec3;wz?<>RwTeJeBd(N?euFDA<1oUpXXT5#i!*VvmMum9d0>$YB( zhvg2g(SvM~BVi3ES&_dnEv?8@D&x5(>lLIE4E;${z&Kb=l=GmjUcZio%%DVu*cn#2 znLvB+UY$>k(KI-Wk>9zl4mQA4p|553#^Ccu#wu{t9?*jU(vQcqUGOJqIrmFIY6yXiX#EvAwICOQ$Zk z`sW@Z(#`Hx(kYRlGs=GW;uXyj@q4Y+8~x6k^kVz@Mp~gUjU?hd&pcm{(8gUI9lxQo zN(PVH8t1wutL54032;fvrhW$EQ6d9YVQgX}7~Ho9R3&DCa4QW)ybRaje8W&2(vUi_ zB4~ypJPFj*xK1AHL1wrQFKNUmfF~?tX~_&Eh4kVGOdOJ{Ft`?6L8BGUOcVmv7${i5 z2;mm#(&39ApaA%Tj?D3bE^)Re40+2=RN@S>DWMEFmmN#8 zC=U5a-O3+sTz`jQ7mJ~ z09}|?ijIPbvsrO!eUGH0mHK|cYd8l&9T|8rfz|dgPEEJWF7b2H^|xuZ@OB=+uK^Yp zfarTbd1UV8BqJm6|M2$KQB}2D`|t(}P)a~S1Ox>Y1Oyb3R8r~MbV!GENrM8?ASx{& zAhqc(0qI5o2}vpGQcChS7y6uY-rpJD_`W~h@$E4fJU(vsUh7`>J?AyA8P_}Yz<6ie z@FJ$82RL#!w?ls}vQd{JU%thgB^JatE&dsQ9@7nES2K)29K+;B!iBctnlER2k= zsb{CBzqf>4MR~#@q*!8l(|Q~9vs|I!Zm{xMuCzMH-B^s2$?1Be$osF1RwClE@E5Ek zjKYeGWv3lZP2)XqJKBj!aHcF@m$O>KPhZ=zT-AO)sHvSSU3yVAZJ1nz$BFLV1h>VK ziuY5>v4>mBOvYULhCbU03PGz>@;Tn()1&$g45@YF?P~|s$Wut46wfhz{8hYQM5atp z1l=C@HL&cBvs|u}LLVKvgzKWfmhA2^1Te*CP{a&EtLKO2+S=NlDJJEd2iYwA876(0 z@b1HylCm-`kW=|ITIEP^6nK$!AzLr&2(tLdFbSLB1iy5K!h&ZnZR;Ic%G?Ey_YRqK z`UkKQOChJM2v>YXWvW=2mHVIizsHrZ`Voj#W@~lVmBx+0fqGM7@b!%2tukEf175K4 z^3l7akHm$J$}`sv+UpVI%XkTKdBha*8hw-aMlP%JO2woCoXY4x2ZOVjY7iOp+)kAD zfd>6Gn5J|B1)X{!@84g^ScdV&JyR9-!7|}VqYkrOR5ugl`wtO8)V4Ml#Bk^SN{{rL zH*Xb7&RPinh=`B3SfQ0A%UfSX@xGFM%vD2JQY*w%G}@q02{O6k5vN0+aq>UB@hG%; zl&Sgq*N$qzJ@yoCz3s)>y~*Dx9(373+wzY)yBci+-D@`<#@)_Pjunr#3BE-cTCSf# zUU~czg#S@adp>x#ZRJx{`x!wT{fpOVZ!(?tR}Fjg+C7Jsp63E~pF&?!(dNro&8uZR zcSJR}FRUq^{-i*GQuf__)vnSfc=zI;&m(L07AKR->_f&mS#7Uq4u1Ql0~IOU%hA!% zJ$hj2lZU%4dRG+24pSaO$fkO}nL+avX^Etf>Pq%6(35>pm$To@E2oG;-DrEvF_@#u zqM#D#LUUK{s^vYiC~nf;{*~HIZ^h)rR+QhFhzy@kNpa0q^GbC56yGRAN{SbYkr~Js zW(X$VmT1WiIVUFiuJCE17qCC*H($BAI8>$Ieiz^tE!*OypLk5$f*MNh*abgHtT+z1 zx<;#NM#L@75;ixjDRMqfE4axUje1@6-`~O6G1~8P7~{9LfVEKW{Dn!#-UkY$7<^OFS-gr-RAw^T(utA7-mh zt#)sZW%_hHZ-z31pwXY;4Dpawa(R(97!4i_19KL#r>3W`@pZMfvQ;oIb1w4}Zas70 z+{ee)I{<`nHd^vDfD9s!jeBu38IgpFHxeh;TOO3%8THjl;x#O{Zrh)34=LZg+Fq;WW zQnDXVp?S)p(>Y0c`{MUYU$fa?TITbUA<6pJG{Apl7xq_At zRb<*HZWIS1;)|PoE8|HbfdzaNbaZDY&wrr+JRvNXSd?{}gZSICoCm782IZwp$}07_ z&5k>pzfNjwrRP;FG$zq&`Q{6a?QWKpk;t&9+)IpxK%1YST}qV#h=p|kog~R zh(0L^WB4?5FoZEGAh@<8iY>Welj`#2FU3!lFLT@BR6bkfEUPBMKCXe@SRL0+V}=?t z{7ZK}1ykP3bJ?~49c;l|4+jTMn5&|nCqOJ`o`evUZ=@epI;;F~1>)(l^Z1Kj@t(VD zk=Hnq=#p$6p~mcCsOW5V-H^#uVp(6?Vx(A?Z|M%2xu200wLSaGBM-s8DdHHl9+nB| zA^NJ4Jp=|@mkYQ>`4-S3mY-S|9u4-gej98o@hr)1PKyyPjn!|?rpqk~lsKKE`DNFTSvM2l{$nW&DRaWRsrYi}>TvZVV~Yp=Mnb?#`L@s^9pjngIg?DaTE zxoUbU6T%MoUno>89+4Cbfgk3u4t{jvV33Iq3JN-R?$R9`0C!&(6XN4TU_K-*06U~I5nwRX zdm$B`S!%_25Z8!7kG_`E3>kuJd*TZvm3Yta3;EuXl9xa4xi{{bG(>;z<^>~ukN1~k z7=$Xfw`;qSL5mOl_!hH6;8)X*aU0aQnSiccTxr5m2!+h%v>V!#O(Ia84z$fx8Q}-h z=BJ6=-L{E81biEqh}WQMGa^%9Os}ERC%?BA1#PdKO&0drnTCQX*noDhJUl#5o*FE^ zk&(*&i#1^25Ybe-h{lG7_s>?x{QUi~WO>P=)UrikDme;i3xPTCKA_ZCYaY*Sk9Xqq zxd%2j?5Iy5oa*G{1ReVE_0^y|gb0V4nKM~-IWs^_S6>q$evkd`FRT`cfNoX8)lqVY zwD8KqgW=Y2Z{2k`;&wXKt7IV}?Veb>b(kHiJ1-$LG5JKf^J?$HmIv$#C!1(SN^GLP0?s0 zS;BnU<=A8sD#9S`kGKO<3L8V)@s%&JXfBQ`)2CasYKtSpHXS1&lMg;v<=TPb=q{UO zOvFymq)Gi9+{lKG^tLMHz>4?YF2y?|l>!nR2%c3%DW%Is=L`DWLYsHgy~!!^yIQti zFI+XlnPdND<)G?3bSWLg|(B9WhVW%Iy0U06~k}wSgw5M0lj4MkmZtd}mdz{7rWZ6EWG0FmxE_V}?JQwzS~6L@&#usv_;&Wu>|8 zwUyf)x|xc%;xt(5E9;#T?d!Q6H+==d!j<2bc9rske0MdwT%Evv&O&@|f#w=wY{PKH zd1OxRI0$)Kg5P}!*?Ssg2egE!1uwhJlr{4T4a5jGl#ch$%+ad>U=vn!I!=A4H3Dz# zg&Y3jD^Heu1{PS0x_H$$x82u8H7aAX2B;m_@D2pDd;nLxH*llj2+I3uw!6)nnlp%p zQn_=88#Mo|zvH+EGzp_yfSJ*9I<$!TT0J z@zt-NSBT|I*8QNuzc@gmS1h^_}aP>zxg> zF!wWsLR7`N^iA*jp-kn3v5}YAi9Vz@OL${VcF2kDQx^#qzovb}qEa!Ocg%YXJqIgR zv^B11rW*A-!46=SA=?qoeH!F0JrQyXgYLsKI(8v5E#y-v55W{Tv9<42 zQ#N>b>pj~Z-_1HST{k-P^5j*z+|b@cjt9*(-&5^AP0@DdnU$_bAtM=+d?R7guvEb@ z3}^m<0k~1F{S6^Touk(0fL$nlsfPAcPaKadBgj-V1+V<_p0Ayl)d}s7A6?I~+x*(> zS1sAQ0ZEy9LRJBM!s}-^f(n+D_77+XwEdeBRzn&Snb>|%BNZvcs#o;T%pwbd*9^79 zMygEe16KRw_qlkqJ;X-?ZY%XAy;83qF24!hXD+efI4^Sv_j<|uE1S|F`6r(8V}7{e z=2lmH(0brKx_yTvt-Zwd?m2;H^!<{Q&z~G|8-s1@1^MOHp$gCL7`I63F#~(;c1~oX$2y1DIPYo3u=}I@G z>;syDHMI-HQ(l@ZO@+kKNHtJqmQ3WnQQjD`WJsG@ml=36vD3^p*)7&_Td-GA#8gr< zpUj@`-FofX*x!5R)0adSH>8Qpp-U7Wqb0Pk>81$hZw=&_I^8~|Y6`CXIX$v53#9mH z2S$sf-P8EK9a)oda;i6$&GLTsO{f$s-rzrQ6ZlFvqC%f5>yyD$G6`qLFnI37QzoBi zQPaT}=11QJ%`PAAKH*(5&+6p+BY1+UKr?4k@@Dj=YW5{*e$x%XTv!%H!n*X-NjdPQ zx3p9-3U^j6?-R$<$_UPUX8JF^3+H*}E!?;S)uRD)qayW2$f1bXplSDkyRX4lRXm?RKjA_vHpZ z7;bVmB%g4@(B0Ki|9w4X$N77ePE`jtlGUI~YLoq0rS^JrrT2U!wH7^^+1f;zC^A)m zKJQK3Mb~rJ)Oh1AAx{(YR?_dxkKQZn4;kKIp~$)QO+Z-yg#yVNLuy|$capCk0v!|jdUbKX^OFyPBPGG?cz)?@fNmFmXG%XbJF2 zS27B~fe0&7XUQhA>-pfRVuqr8(h6Fo0vdgeC{LjsFj}$C(uj@Ob>r=VG9QyyHWFxt zK%jH*d^+p0*D?~>VKP2{Y8FxLyvniCEjTbNz$tB_3&^Vt4|?eLqh zl+K>^HG8ss8llA5d)nwg2_jdG*wIW=+!x8&m9_jqFn6e!%|R7OrYVXn@f~!#_H37J zjB=%)wYPMLw{j&h8{L1jrC31Bsjt`e4!mRH1AXnAaQO>Av&3&fzsi&Xw z&U+)QO$+u7um;CjyZ2hLiQu-u6zB^pX`5^VLDVP=36Z%){+AHAnwSLzBnn#p31^gUO7~1CxOl zPfz&7FN0k`HY*+GoH!5Rjb&;Pk_0jPh^H7eYRRI86kla$w1lA+9s95I#1tMXM&*Kr z_|KYOuGw0uE`SBbP;t#%45vujr;|WawqMHFU!flJoMWb0WR7gj{^I(R&7+@kG6MoRCOZ#v4dS9B* zYcN(~Qm$=-qs>&k<9|UWtqB}@VPWXv0j6O6D?F+n>g`QyZ*%*}o{+@ZVhCc<1)UbLik>lK6j z`qWn=+2+pj7%@=<X)Sn-D`!Ct1{trGX8Nfp576(0~EqxiOKoWN?F!w3oCfQ2ls)lSREC{zDaPo7nGPf#f7NQeL&)5)V9o*Rz>g?heGakmHl zEVMRg48(DFyJ|MN5M^Tbawcu8aZ}*OLyJX}ef30ap@3w?$=`b8ORZ65b|li;HOT|+ zj0>E*U8??Qg@lJtEn!IhN9iOwcB@K_x|S%86FkSHw}h@CX~0*XW4A#&*;7v(NuBd7 z_d-hCb=TF=w3Hn!=1zbnhWU#l*O6;2Q4%k^-;Y#$l=~`9tx*ADAY_i#2=Sb)e(=zc zjy>tH*MzC%(WjYkR=SCEZo67x)x~pY0E0=e=M}#oM`y+Advo2F> zhr61CR&!ZeF1n(e@rO2i{1kjB5@+8><5~!BL`P>+WG|+-%3{1U3OsX|ntd*YeqZbL zjJtt^fpW3_3_UITYTj6t+B7?pBhQ(+YpQp`nX^n2F)-j}z{)=TjO8p!Bbdi8_t_@r z_AArNhzFfZWGF3oShy}>rPFE*Ijx1FnI=u4orMhJ^&ZBSY_pC#b$ct+YQ7sSdd9!o zoPn9-*tRky<$20i0lbR6j+jINe7)6JwAm9oy4I!r!%> zhTnTl6O26Un4BWEFZJ~41SB_7*G#b?*4@autbrVmvNexn*{F;?spO#aBck7nd&bYwB@ZsnkS1&XR1R(a<;7LX|#g(cb?+7 zFB{#+TE@5?MYJ^c;i`{O1%|qhdBJyY4838uUF%S$+=xhbq%5~v`&PsMRcya!U^JTXC0zUkxq zU!GPTHuDa0?R~03Czw*o7%tuauwO}SL6zX6P+!tlGCuUFN&Go$GlA&hOZ&BzUCkS^ zWYH@hW)|YEo%?OZT;jn+E#>-K>?DFRiP8KJQ(m_lD)^3-}Ly zmWps@E$Lv@K0!kIV{}OUz1NxxPkwds$5>+pzF=^sK$TBKZjP^lml($Na?&H8r!^hk z!7gr&8;<#H+sf*E0_<#-7S=K`@hq-P#{nYXHZCQwF}Oms!_QEg!H(P z9Ur~Lsk|}(ADKdW<4gT9h^ITTu$d6BU&Qlh$Fi`?#$JayCoATKSn9!P$RneG7L{Zsl#Z*&!^6 zx2B)qc)wJp@!wfrDQTL0*jLfaL;ew%9Mk)u=G^8XCDawYzN48)Ho5OK0zru>uDoge zvJ+hnM7)?{fDrY>$mQtp=C@MTt9x4p zTzB!FS1DiR`fba7Tj8VT+w|E9`5)eRs4N1WMg+#gtIv@hU3|@`@d^vG2J3|UH#GCN z->rV)T=KfdwI1Mg{#0Ux5!?9OebM9|7}&1$U~9RZ+uZ3fTiMSkCga~fFWoR(%G5u? zd&+Y%XqOa*ouW`OpZD&G)#MwoAti;+_H@?fd6UGqv?=3)D$qIZN1BzYtylk8MGQY) zIoZd!6W^CFd-fc1HHOy+UNjxh=-U9|eq<099jl$=K7Ze7&~*d*rJXSrCDUOJ!ch68Z{0@soll^) zt|_2T-mwOzLqlfgB2=LDsj@_>*0yJ`0leXpg*19fTv}MRoKsxz0f~{`(Mgp4bQ2aT z=lbQw7>_2%6+6T`rcPmV1m_q&L24*kPEs0K7Dau;z8Jyln>2DRuH8jDZ{JBxN66v4z9jTMLg~J(DXPXkh%ts#p4x z_oy>>XK$n!&r*yw5~&!A`kHp4;^f^Y?479piBDANnzF6?RQq&;;3}*UHlvusA#+dd zF47+0C*GR!_{<$Q3;vwLjR-CECGB52i<4(Rmw0vFW@pTXQWZw}MEMUFb{40}_U(V- z#SCoLHSkm2{wQ(#tpjKH?Y}zK?TYef32FD3F*h}E;+f@`MosaY9b+k)A2TD~Xl6ym z{{+9+f0a>{;N=h3d{u{BPIBYhJ}Z-+MEULJ!E9a;)4R@xmiKeU9%DAld4`84%U8^A zCqBcuwCAk2yTyzT2%k(Z@?Ss1QzbTJ8R$o?0pVbpOeXw3-;geY2d~^t{sD6#5^870 zB+9rh%FrssV$@9amXx-OO&Ju^JHTQB1{Ao{!VBPF>?iJI$o_|c)Vyb+Yfo!F)!utx z#zGeo0TY9lfxRY#JL{ZL#fr`4YCEy2#lZ>&MvEyhFO>M(Tn8C%_zXcEbV zUTQaLTu(uk+__iG=l}accZ~t2W|=*1DN}-+GJQAj-@p&kD;0lj8Z;2}G3EV-9%QFU zWD#r_8VIwoPSA=Cj-{b?I=DB;<#Gb`6@#sz#ZSzfx{p(i{+8ygyQEazQu&y?Mqj}{ zoDnJU;W57!Gm^>aFVo&oaUv~$<-18knnAd!0RBe}w!~*HK`UdYfglp6s7!e>@(*S_ z{!Ont5|f~&={=oAUNv8jDaD9jv8FLV4yCd0*4-*h3TG1w38Uf^8#1|&)7|)$6B*g` zWY6|g7@3PwZfCy5*&=6qK2itq2C4)60gnWqrq8bD##xfvKGHN<^xcrpBzB<1%dD54 zsCjxXEIg7hCGNNs3b#t=3s0GU?ZRt8G_gZGK{Mvd8!m@U(TptvD7;MLDXsg!zf8QO zomP+_9^6p&hpPLBvTG>I{$`h{Mn~kq+AV*8Fz!{)5-xP=0q)L|SKJ?qr3{s7hU!^^ zmDzbB?`Cy3wM^YeTr$i$^7I=?s4;jxsyEB>c2uY78yAYpPumN9FKgu5>zlgqsOkjF zsw#4^l!?w%B(JmQgw9AUBDTu{1irhbDl_+@XfB>Cz9pZdMH?Jyn2V`zYm>3 zg*H^hwI(@f#G#qvrGvqH^0Pbyc1nCeQ%be0@FIZlTfYt|op}RVHB0+3|K0`b-o6-d z3;A0t`{Os#3luho-d!pYG3=n2Kbpyq*IDX!HThN!B0%@|%Z{$;p5K`IM8s+&_VG(5 zRnw-aFSkV(nzwGD`M+1o&E{t>QGzuYQr_@5A_%7;%_lYmGgI4k0Uk9VMalG$W^cSc zN6GY8DU-N_=#-8@TL=wR($D@9*U05XGX`n4Cn}xw)(on{d?CT9BJ%HXRb-rI=cVpB z2dnV_V1CM>sJzsFf3=(A9HhdP3yrCbrkl|1O)<2KM z^yWn&;_4rFAEZlY#&PYRhRuRGay z`ddpx1myqw$In^<^-BGf);kS0J+y=>)NA1GL77=w04`a9BJV`v0@3#B=dx-i3;)Hp z#o|8sPrupxJPXs80#(SIu_G-aEVQ(YT-137s_+>$2p;Mo^=vwQ|9xoWA485xm-yK# zKXc=@DQ<2*=7L_7Yz9}R={Lv_cMH{RcXzkQlzR>9Fa_l(ymCGR9NgRAT8#S9m(8-- zQjOGk$CL3xv@88-gk+xVWmu&L1Szu{O#jdWmPn;~o6EV>^u=Shq2$k3v7orSkIUvb zDxqU!D%Zeboo#|z_@M8*UiL#DFcBL#4Jkesm5V=sGd?g4&NijhJVqku0_Owm`q6rU zhni7O?c#uZ1OhpaeN@8dR;(n05InWH%Y&$OlW3J^-|zHOa|L^EWS!sh(F_tCY`k%2 z%?9*om)R;@D9dgxXPSKd14tEBm&`T3v}7Wc$X``+GEBR33^Q(`nLh*~Y-*Btgrn(G z|5j2@G$SDojq>#l@*xsS-fnyz86Q3L9PZ^Bhr$N_d*^!*vIWWH)Wb<{_;b=@L}8*0 z@txZ$ZbA^W{weVPf@Rwec#5`YfNMD=eFp8Y$_G3VePBPlBcC)I`z`Q)ITv! zZE`Z0D{IW-d<{*N=LmLqH@}k)qZyao1xkWA5noSd#81W>k6-3g3NXFw)VV$2-@bEi z@-)ksUwumKdP^JAr?sx^wb8daKd3JqgDtrX3XJLB<9cT*om%UiUo7B~^YrfqUjRp* zqNc2nB6t#&eQ8H`V&}S*MYmmtTM5m6)rjQQePXAxtgPcNN-RABf2$LGw=fmG8}ya^ z*6!kMC7<|Djj`)cYp%aC@n=4z9P_8+F7pT}+TaY?l<8L!xE%FltS>+yL?m*?-WV0hgsD9l_GcP&m;|r? ztLekB*Jt7y;&;WZwCJRaal!fXoD|!w+MxPBNnAF5fixI)mryoi`ZVNLdj^}SXat4{ zjV@L__nD#*3)~%J*h9jT!?SXO9<+G>)6be-x_i8Zh|w>@;%aUx*FarT8&!*t-|-+^Fe;hkIldLH585^c2` zV1;!o6@ET5`iX}Iy5!bwhpPW|eswQ_5KZCdnDl?0S$);xSQNt$zZ~zGt*%)$Qz{s# zJnv4)x5+ zR%ukt3~ExIBU9(YKX>5uXW8hG@`LSbRWq%#4)ao{l%k=KvwkcSP`hgQvX?1?vY5+9 z&*rzG^p^gG`CL)8hThd%BET)L$rA2ps6QJ7}}pdJW1bE3ZMe>Im>FDFBN&$0V7Vzd`+?)qo3^pA6SbQr_xQ2xN-#P-BCsmKW2 zV!YCNrNs5#BaW$-Ii1P>@5g7M7c2DCz2|~jzXIY}$NZWt#kv-61xiVZnCUX2-dFpc zJ?3WMs}&jk*k%r&?5hiQG=Z@$7Z-2Ia8=`O?p9$_UYFqr4x$zXW+?6i<$K#FCmK== zbw_J^5yt}rk@!;Sksv9k3%sa_q-@$IsGVCe|DIZun<^K}X=^Ogae1hBr2aTrOqZng z;fGrsShYFY@565Arb`I8+rMI5NBGK`cu&w_fB$nxCO2LTp2(sR7th^bpCZL(ALlZ;X!Rz;ehOWMocO(Ja*)T<~ycDIvR3kPXe!J zA3-)_G_^9ZX?Te^tor-uYLz<^xbHN%sBA?yeQF|OTlpcv_J78?BsBgRq>Gu)fQ?c8 z_XVo`xj_HmH`hP%@Fkb-uOx2XX#31r@8=g}t1vXx_gvFzKQLp*X}55WqJO)w+&Co(&7$Xordn-xf9fSM=|5Xkk&V4J7gwI5Ri7R8{hnm& z*PI8rx{jL9qLWuF?Xxb;iq7WZ{jQUVI?jH%O%wyb(ctT^HTfFz*nJL zkxz$(QLo~(Jv+r%b-_J?cag>LcWsq8=D1KwI%e@*+AEW2hjNjn8X}+nXR>elRzWJJ z{@1C$i6+eg;lF7TBus{|#}Glpm|&VqiagT{8qLwHXHF1aR;DCOI594kpa+xti3Gpz z6vwI*uby;xhvzGv&4RAsuf>WQOydI_aNByGL8I!8zTM~p0huf(^OM*MYX=X|UdPpR zf0A4Od|$L|;-ydopcbzSd6|a#+4qLIT10lSt8Z5r?~PG^_>g+y$ztpT_>KLqLMW)yVY5xw*+Q5Uw19tt~d5$>>6$)&fQ$~>^-a1{4>%w5y(@9y|)a}*r-<3`{bpL zR|CXVv5vVi&%oC7m5_Zx%>6heM53GxyEmQ&Wzd76JEjDYCkr< zAs9uEsT;0>CJ~*BNsKz)#~&jsoKk;mdhz9QKa*A&l;aMBInk4r-s|#ILqB;bM~sgD zualzgQ&P&kq(~nO-K)r}<<9XrGsQ`JX+ClJ%VSHYo|G7?Ft6HH4QdwEkVaEp*$U2C7!2aXQ=YD2#ZdZ)PMIkx*LUA8cG9Qf) z_w3k+-Lad5ZEfwj(|`ZPzy8T&PZ7skd^JilrbkuvjAWSbcT48&eKID>)$V`qaOSux zQVyT)P3yTa2@x+eqavlfnOcq7-rvs(KXYt8?*^#{j#;IX80Q5eZwW@qq}@r{z9DU? z@Ji?Xv*J4XHuo2{V$6!-;9bMK7e-D(;BT-%GB@i#Rx5=u{`;-MyMNyPJOIxo2ns5E zOex`3?&b!2R`ygYF1o;0aTt1ZW12Vok3~pYsIL(O%FUDiBjqx^c$vnoYq$Kf^M!dvR$`~ z{wlIvzK=ZTyhhqm1H?oW-AHZj@0M5uRA{D3_#eYn$i^U}x2KlCU( z$xR)>lnX5#9S!g9+Y*7MQHnh7#fum1=DuK^y~rZ`5NmeeI$Z&&L56hq_bP-2JAPr# zr0o}2<>%+m^<`cJd1DlcfPeteTW@`H-TLSBW3$0Lti;4b&{^SFeck;YMh316SIesPiQjsuTFoQXDC{Gx+ zD$z=6{+WO87XSX5kg%AT9!Rtx{-y5;3szh(gYZ8&{l5EuWk%r%*WIkSr3e~yA(!uC zqsZ?VzpAx|qUzHGmvKq-RD%7-ziK%N>v-e(_3QK2($XQ_n?=J#dvAqta`1jvGGljs`&-q(W2vwgx&=Ca{BDEmJNJ~qrlJ^7)q&t){T1o%C zijdJeW}yG|*d(9${BwSh6$!&VMm+zl4srBx@8!S#_`m=K?!fq-6Y8-~uB$EfR_JXPq`t5+pPy5@V+5E0t1naYGEB_-(!X#~)Eg-Eh0PjaLT zG2FjTfyhRIxWbntk+UEcP}kpoVRNCc9yFQq4caff8BU*bvJeUVwqb>Nw4~JJbol!_m?46Fe?zYU;V6VkTq?GDNFy%kbkDw7o zHNop!;&s197_az=(=srK-Z2}LMG>LAp@$D*XE6K0w|~+6=N^P&q1blyF<;1(Xo??PsNP;+#GEW^%4J0mhidE_r~- z1%*M{O*zK!I*_v3Jpv63VR!fH?!zE+#?XLq?37eis z5GM@G8XOjSH?EEZftuH1QiNW4dAC6}mI3M0kl|dM~!8<4m~nxh*3z1t;`1i@P5BAaYu$v3Cg0$44-c2-N@^p%I*k z50M~%Z2y}IsgJDphAnHtpYADEViCbOX;P^AC6SYp>-h<~e{3c_*KWEu1yLjqlqSuV zlkpk$)@Re@tB3u1LhD%lfUMAAS#7;fS=&EZEHnV5NKKp!4O2nuC>x};&ZE3bOG_U~ zNc7IP+iJ2$1~f2#w$Dg~8QJF2x@&OA6c=~We=NeNaR22u z3{tPD6L_8E)3TB0c;Ao&ea(Rh+3bCw0)C6H+I}%Yni6?Fj5jUd|5qrgCM74Y!+6%Z z`uY>v9tXv(n@;iS_EEBl=V$)2VNgSi;f>DsPtQU8&}Ca`c+jtHac!c$rL*(Y6W$z3 zPBTB~B{}g#LtzsDCDMF>A`g}N(@A*ZOQ=*6r)m;%Se&!&CpgNMT8w_5o<8YmEn#5v zul7Xc4c^kEW-~g4phgRkbr3Zl0u{#(%~COU!l-|NC1TgJR+r-)NNM{ClgQ}+V8Nv5 z7sqw`KY2?{f#jsGu$jU8YI+y-9-XNqI?!9x*w{!*PoJXJB>DkS2mo0?u<@X#mkj;T zD&rC@9!~8*J48*oO@3hgG>MAV8y>tJT;R&O{7d8*p@6 zp3q2Ljv)Otv>c?)7zoD5_3-yB$`hJHuim)v$n@{Y5vIR+)1f$glb8JaZ)`LR^n3$n z3N3$$0AHgx%`eF-D0D-&j`z!#;G>k(B4NuZyrs7&Ue}21n1KE&@L{KymY4HgfB%%< zoN%(}15+XX_J5#1nj{P~hx>wym>Ma!ZE0^85*H6p>>q>`LVa53%S5IPFKqNS7(Zsq z9_>QzvRjWuFSaAaq&IcvXFDq(i02#3r2iao1na zZe;wvkmcCuNsM5_GyE09Zh+U*+skIub@>kTnJBtB#;X@2(OF<0Xdb}+ITOnJ({BU+ zypP~>F&Z~gxTabK8NO2}6_AC5&Q-&%#Ph^`rf{}-!6!Wb^y$I8Xe^>w4-$%C1%jZP z357y{%f^PCEkQUno#n^%)5l8gDiwA­^YIYRx??Zincc?nZv=%6p%+S(d=O9f#- z0Mh(XX!JQruQ?>@?$d~JUom}I)u{+@dn^J68zc@(gU>n5hrVZ*jhOgTB~Ax9CGuzlD*cf_=suTV%e}T`+l6{Ci6r zs0Yxt1e@7@h@w=HY|LqvYF_;S{>@zm>38X+(gswv?ZXKzuX-ACZqmoWAz6{{Ti@Dh z0!I}L=1d4o&e%sMc^jhda6lweC;LgkE2I5ET6tr>H&OORo^B%^NbrVNuzt49ya2Qf zoq&MA{QB~6X>d;3XzHkM%R6jrY*4hrMS=aSDAFKF2B~5&T6c%ENd|&UFf0aqKe%~# zgfbKtHo=!l^}+!4`TVVQ@-iA<`o@nAJW~;p(f4gr@5~_6tN-)|qW>7Gq%+~14ifc) z$IuGn3?5Y&>>3K`TlXq@2fO?YMc{A`TsYcid(4HfwY3$|TK@vD3q+LW_rWneTI4Sa z0=~Gz=J)FVmAJvLY>G@csG)h4^x~*CGuU>gJGzfbwl9G|pf(jvODngu7uj!oYW}otZwgmn~Kw%{X4Sx)**ALM~ z=(g`+N1%?;4GX3KxN0p#{MU1}dXE4E)tvbUzZ*MoD?w`na_t0*H8}tA%B>G5Yr1rP@tx)sVFG}p=le1I0)Qx=iXYp zFuuiESXgL1_k|XMPYk_0ZE_`eMES*Sk1h^Z7@&g7`oE&@@k~HLA;Lb3Aj51 z0XBmYfR%I71%y^BxaRu2O(sP?I;a*@)^Op@P=3)IMhiuV zSxuy~1|){bF#H_P94AD63{<@Fq7_wCZh$8{g_0|^(43c2QdGPKO^G+&#l@XMu^E54 z!V&tGngJev62w50d#g1U5p?)i8qh75^F})P?I)fawQH~tbDJRXQUMl_6eYa9GTI2L zDF8EJSNoUS&Wc}`2jS4%{#lV?F|g8)BqiUvFAU_uT0xiU+^x?Y9be}oBO(}F-P}&0 zpv~|sLbK3%6G7LlE=W+;H8nLEYn-heObxgxGeXT}^#D$~SeAOJG<2aUsA_8~aD$48dKCCoJk5BnIoxzz#$P>8>C zbauv5S67dlAz_dt^6~LGT?!qZ1QXxCU&&saoTQ7JHr|cd<-`kPi?Q5_-KDr^@lI>} z&2T))=WZ?4Z~=2rI%i;8@l4d+z~6fJGPqi)0w+>-(t|8u~-f zj|+0}z;ay^6clU#BZ-7-;8O^r0i5JJuIsWnZx|!bd17XU9>jONLklEQL#$=>%@|H|MkKJy%*>qY z&yH2e*YyFI%wF!%#2GH;G|74M=Hpu`c@NIf@N1kreH!oCvu8S>{|wt(1k@B8pXOFE zbR`IAV-6Cy?ZsgriQ3m}b!E_L!J(m{VY8c)CBAe?P2_Xbtx>Owk0cMsD^=vWe~koC z7tT*8gbikA{fUGOA`=o=!989}m5yT^Dz#(<%E&u7I2fY(i+}@&0L~GEp(6$MkKV!; zEKnTd`BFfwF_=5JWVC<+$bgE$`Zpy4^MAxGRGoar8&2)7^gwp zrw1g-fgz&M2ZLF>zLF}oBr zE-ec@puhP7vzj-cvrlkk;wk5WsEgur;e?q*?Hjz}Js=DO;CN}cG`FYZafS2d0)3;l zsz*~O4Xb$?n3B^2);2bRnM&EW?H3d^s$IWLI|3XHbo%up5j@2jK!V_4{Er_$eyMhI z`3Rt*%wajE)_tluly&J0pf|X* z-sNXNzX@8CJdO_T0S5Df&wbnuM+6B?_V=O6?EOQpJof`iEQs>L;oS zK<%78LrC2KL<94W7a{Zr-wjKHLV+Pf+SHxS2>0ax7Yz?YOi_^(&aeQm?+-1hI`=-m z>bZQ+n<(mz$$1cd!AAi>1Tk()OA89c#l`icH&u$9*D_Q z9j4}-LanW>eS(=thtcU%h#A0=2fi-3;{0#OkW$&+@m| zV1yO~{9r4XK%*J#06K9wOb21*4iC1qt7brL2wd*>|%3 z!NER&Uo|u|P$-BBpio*~18n$Ih9@<2|G*mKe` zj^B;DdS~)1c>j!55U74W?!G5w87m?p@+jiKS(3X&0h@i1c4Sunrf8C42>a z^X-wWSFc?8^j(E1iMxx8@hOcf4fGC&2OXEk=!d!QE#Esw%}WjtQxITPKB&M05&?-R zDrh_}gTNO?B@r+&F)a-hp9Q@&Q9;2M<-StF#T^iqBcKeNCZ%L5s_~*XK*#6EC@Q4I zA2x1VQD|=UuDEz=M?4?p{QUeQ85uE0M=ol&4Y}#*=_k`IVGvhx4j%m-O9jkBFug>? z6^Nr?nhTfp}VD&&UCol7xlwIz#wFWQ1{}nZ^UgK}oNQy?JuOgi4BCWEjNrK5Th=pVT)p(wk}u z25q2+Iy$telF=^!UMz099BdA(ck!%Vxqkf&fT}u}rxO+%8wi?B$QUsoza011n-O6i zP4aWg&=@qOWR;2wW9{)Y9wYiYeg> zvk(eLaaq&B6fl5I);~HZBxCO2f{E(k2D?&^3_{6PF9pPM8c;MjpK}p~7_4|V&^v@E z;1`0r?SYi@f^3cs%*R5;DFAV(v9O$$nu?5gz(OJ7eYQV3p37&PM4dc!N(Y93!5J5U zLSZ=Y95CrWUw1}#Wi5$<~IrP@f#szM4?~{HN$`> zBtt>*EmvE>Z0sWi%4(wSRF-OC>etWhM4dh(Y4Oe&!C~581*N6Uh^iaFCafDdasa{S0IiVg+nGxaMWJ8@#dTnddl@MTe;n+= zM?CIhCuBM{GUCT&HN>j#fIq+iCy1zhrb@+rpP4bR8-?R3aZwM#K?DQEZS{;bpbJB` z_#R#2G`k^{n^+Z+SpH+SBRZMGat{U#bQc=QAV#*kn;1zb6zKbRTRJ}d@sZV}BInFD z7U?*T2*aDlz4)DRJj#~u49cUOEi71&;1)6Dot^kVlL9*_2)OSyjNX`^#Bg@Xx&|14kdhG?^jK1|0WiKd z%$lnMyYArNfQ*7e=77P&4IkV_SPTgG0p=2doj^EyKzld(+?d3ZevcZb;_d;MMgX>> zlN0id%<3f#*ArY{fF(a)`X)NsA2@bwV49voS^x~1T#dUM@7_`&0=jGYVLBdn%~5*# zjgK2W62@QCiQygXckH9LRtLyCs|bq5NiM;tKx9}NxRX;dGBPl6uO6s8$TX1OH6;an zW?)wd9})$Y3DL%cv?v5B$lN)EG5~HuC-1XYu8WEa`AY(-063!$5@pFRC$k|`5Iis1 zH?3h6Y(REVHe-)(XOGuz zj_Bn%3TzNS1cE;aMs7HRqFOTn1cih!5YY-^PhSG6bJA3s?|)wQ0?o6vcGj9FON05w zy{U+AeDd_^Ss(#Gj7w2mDf~w{7zezoVj;rt8~!Q~E@}#xjn}F37?a(tWyC0iFsaONdkLpx&c5xVVeV4++=0%V@)n=0u znBUjhpIq44*@fZ3rR2BivxDLsBY^o22rKV=03qusRK9UH9Wf~>1j3Y*l&~1LHs7?* z==NdIr?=H1($w-eus6{}31%qb`3TY;2mpiF(9l4>bOf8}tg^DQiRRO%PbcQ)ZXAq* zKhDyu?0+&2>6!WckmcoN9|#&y-oO{^0J`~5GB-CTUlno*GGGx}g((t|cu4p=SCtBz zh3w|do2N?=TP>HX#b=@k-_=<0tuL{%?6YUK{T&U%rfl;kDz*Y%$}j z@N|N6IKWE4N>kpGHYx|`1~$9*_h>~~84eP^8N5Vv!nqf}4GrNz;5B~)?1(Nbo%2DF z_O@m9Z&Bw_O$`lX)yBvretH~jd&uvH`xu@}`E9!d-%W-R20Ij#W4pg&ZQ=sFNx#YB zNO>VVNkVF~q8f+GzP`STMKCxM4>bu!{qB&*;r{QQJ+AB5o2PBr_2F8uJ;}MOo<#4O z9Puk>snAZ_ltb+=SR;|^FUMR~`5=NWH0*rcwELZV+-*Cq)=hDm{KG+Ie^5IFr$*|? zP^i|~5)jXx=USPZO!`&cU0@&q$tcy-fli|3&qB+ex%KX>9=smopyIY8LJ0#TOEIW9 z+|4XXxcjvKt*IFxzqh+c&Z@z_IMkA!Dw6sRL#9iYZgzCw1%cQ)EriZ z`zTktrn=1acfpvuICp}-zdtb&^P!Oa(+Ke7Tu9&Gp4IAD6}KMDpXvXtao64*DN>F- zE(j66FoUFq4jqqR6N`^pLY})vBo7jACQc&f84c!hA3GSce6QLXdP+x2`w;SP&dwE4 z_blV>bhZJdd|lKY^A=a#jb*nS1YS`IY zTNA^X(SfmgQSIpRaBykhfA_i zS3pwkdO$#c-EO>qhgL<)Ari!b4?Y_%j<67%oJt>e7yeIsZyt^H_l65UN~VZL(qJem zk(5$0MPy2nRAf#eLMSt34rwry%rhksN*Oa%B4fsknUX1Egmdlso!>d{TJOK-y#Ktd z@A|H#cs$Sk?0w(YeO=dm?`L%dz#B33B0F}-2piWB846XR8(!ZUD`#+kB@bhF=B}hQISeY45_yVlTu@&qowfa zot*OKI+vGr@m$F?bg4v2f zw?;4RQOL9%{<<#eX84bQL~V$l)7f%3Y7Up`oD^PUM800*)YS z<2w}`92`+^SpWjKc)yQMPs`!_u$7Bi)I3i``CyrF8XrJPt^S}gNi%^KgAo15Fy zQ|4ztMuR$}p;qJH{MR3L`3!OE4BaUP&Q$z%`Vk`~+PXOF?Rq{QZKuXdS+6oP*TJpw z)#MpHeK3@6dJ4_n?TWFvgEUg8F;Az9@7i@z*t}JpGT#{+h-Zq(i=x>N3P5IQaE)Sh7#1q z(vy?r?}lP@>K5&>kGrBaP)boDVF2rbt?T?yl2X2|%s-+K+Cor@?!|R{qobp8>(g3= zKZ=i-dSlR-i6qLYwAupPAzJ4E-7X{~3MWtV`rw_yaxk%#!Gm9s)7C2{sG-WvH;mfT zn%;+VGTnNPFBs72K~^<(PV8$H#0&N898|dxId`|aEC2rSEFF(4>2wtKM@D>^mPI}f z#mCLf?Z8OW{ZA^%7~-qY8acD@5<+huyLFU=Gc^U-C4XJhUJL2IQs4Dzg_z+RYGRfc zG;`_rqxRKwbTtY^CQ#-y6@y&C!t>5rbrn$!`EA`H=fD2~)ieE{&TJg0`I{~nk&dAf zk6AMHo!jqWcsRfK%=q|jTS)*p_qP4CG9veA+(4Wb0XFglf9lGY|6JABxRHXfn%{0r zkHJjCc=qUZWM*#QQd3tc4< z=f+V$v2AMt@lHhnxUbLku0r}}ps)!UP=~?P()5K)3%n&O>x@uX7f_IW>hXRm&;a$D znJM4<`nnONX()LMqwnNiKeN9M5F>+FRBWr`IC5x+ccAgq`m<1-5=JLc6L#T@j89FK z;&DLUupey|n&6a~-ML4A%VSDs_mG6j#)MqgyYB8XzHp5l8kU@U6Tj4I9^6uxH8a*3 z`20CPseD{qT#QE3Cy?)k1kUc>P-g{4$r|vKO%?sVL#t5J@Bw)R6*^igGe= z`ymHr0ZLtd(f0UUE4#~=F9*DN@|I-Uy17wt;*xSPWeLq|_v3`Pnp_(#+2PfSjW zO;8gD2+@djUdr_ZR*ZI@tMaT4`ZSuKCzO>xcH6lbQC@0P}Gc&v#ezX!StFu zNleGII)XK38ZVW8uCA5?4$aHYC)1_jgA6ilCyGw(OdUozOJfr;imLsOAG0aCNPNv1xb;t&XxF* zcfG|qeR2^moG8VIk&VcFLWsVNVyT(+E*!gfc{LW6Ft3n_i<%ORk)Co*IH(*DJ$yhS zkis7|qQfx1_S+5E_oDC)5DygL(s^esemWcD4t5FW`7S)+a!9z3LT$`Q1$sOC5PTRUsMPMv$!DEI83|MY1Eg$MgW zLWQrNnwFP^un;nrrY|EDvp6!8De;=t-qA5M)%6^h@h$ydq5?_%jnG{VwVrB2;R}>6 zWjN;v-^0B6b6GcS3i**c(BIE~X|%gr|0Q2?mN2~lg=cAr?-NED0z&gXV-_AEK+c)N zOfbd5XjTwQxHLa3NV%UX_9-rYQ4^}GLj*8cMBm1vG*o55yFlfDcMQu?PEmgk_am4w z6oYjGXK&ujDfUn_DK|I%*;t123ab`{f|;OD4FHHJ;to?qBu3wR6!xVnQzOh+CnX>V^=6MwCf@36T&L^>up@yr$?#lmmf3oKg>UaEU!BmjFvA9&dAXGX<- zgHpPP)n#D=9l)Gw%Q{RcgOgtik{9&O$r#nCbG8>aw+)JdN2y!2K%4ZV_L9F~N+JBP zt|JwFL%QtCi?bYrAx0F7HmVFU`k}Rr?*^k0ZK~fvCUVfd%AE~UA=qqY;~mgYRb7y zyYK|+46R)zVh<3OghIskJ!jO`ogFJdJ=z7Gt~uXPgw%PsRaq)cqZeEh&?GMu_?Ot{ zimuziJRkZu)8+CRhuKj9=Vy6YvztGD`t%;oJ0{9tNB+BN5r=G!Eemp{7x&FQR;!a$ z7@47cMkCzIKcw|4_-OS*9Gb4h+0F{W_l1Z1!;YrdX}k!cF~hv=u=}E-<8h6J+-r%= zLWj-IDhNA2diYRry6T2Gf=1n{*}8l0@?dz0A4*Yd^^R#;wY&4$*(ZEE4ln-sL!^Dk z>#Hy;69Y2yY+QrzmbQ^nD(kweLF|E+=H?>g7`IW+=1*T?ZvK1Ab-;_bZ!=wGs#VLC z^_?XpG7kAl^}72)x8!xID&V#QhG>I&=W_{nOWSJWPOjauWi@y-6zPT#FB}?qjdER8 z&0}(t0yNXgLj<%G8wa|I-PL536zcbDc+4!k;T!Y2>^~#4gal5^OcyU++?>X6@!d@U zP#E5rsY+z7ufK+T-v4P_&L@YXlo0oy{yNUXgf{z zpE@BcyV3I(4;7L@0pib|<=#JKhQ zjiaqO=6eO2+B*$93!IH2@5t7_PEMwKcXNdAtS;CBv0oo{@K$JV?)-2rMg54);Qivm zT-0l}i_uAm@DioBN?SL~g5x~T*^Cs8XO&K;qhZ&5ej!78ew%4U(3OjqE`_&@9g1H( zwUAl|+LKa*D#x<7oXLIII8pJ{>(?hinlsKojJRKG@=kaWf51~*ez^=q!GF7?@Zvip z*n{S3ChFyjXH(T=?;Q`D^*|v|yC)pP-uVnF&1g8$6n#^FPY%E&)putjCI*QpIcUbBVp}=`C6|2Q;*3Sg0U5bt8&uf%yb5 zU{EhS1=>AxsDtCT$gGqiG=a$rQlvaOw~kq3vp@SG2FgoJHFo!X2lCzt0gC7Pt1OaM zQBCIe0YL}v`(%pgEEp(@vO(CbXZ6GKeOcGkjaGT`9)`okU|hNzcq722t+l~Zy~^?I zu|yIozC|yfwio^#FqrBzikv}F71LHZxB0LcTX=Xlr$-U1u2W1C&<1Z=H$Lw4^|wo) z_GMyJIS(FO56^7vFvt4!JH9awd9vaT7LLD$dw98@9yYJLCh79$O_0r2xb2@rwgEs8 zKOC~FqOV_AB6kI6m9s43CR4}Q08XL`T}C=Cj-OOGaa?__aCxEd)3%qZ2*-Uj%9(~j z2XYU&%r;TT2|k}!@zf=djI@jzcZ3P@ASMpx2lAx~4=^CPfPSM;x}e7&cuWz$H5zX7 z;ui++2ERECYZZ2HPeQ1UG~|v*=b4LUxRk*{{Op*_*@_DzJ56Orm2kb(Wg#-D>A|X2 zkXfp`H^8ZKK0Oi}DHx(wa4WXaU%?y@A9c+eD?=+x;dgIihbBSc#Cnw?QtQr$M2dDZGc#CSfp8h%jj`lF$+(H);pgXvS`GmocFNtLIKXK* zp>l)snx4{wq4+h!9BbDSYKp=M3YYtON8TQM7m83UhTM^3$CP>j7n5|BnI-?gK~6=t^felYh&d!EL!jJK;Im_e`#w>4; zrnACRnrJo*q^C&7sQSQ7FgSvilH9r+UAyj8RuDZ>&=;eMULd*8&<7fDXDo(q%O{Ot zUlR)g|A_W{{D6aaesTwmT{-M~YocvHa z1bM=LAWS&O1No7lAh9mQ37?a*2}<5WmvkID1E|#RtE*KgET_e%gZ?o$;H0UfzykdJ zHK=7%#KBz=ql3X zlutS!wNEFd_uJ0S&ZgH)q8l0kn}PDaw)RZ2I$zu{;eiWCL}#RcI(B|k{^uhkRTtGTQ1Jqx-YoGh)zubAl`v2#=ET-x1#LP%rfSp z(CO;xKETx#FPBj)#Ltc+ZzTwg;=|o7?&|7Vr@K<7VE)J6f84h6VpeefY|7CkcG1J8 zpkS-@{29(H8L9$M{_`0HZWl+c@_>UL?Apz-4gpjTFlveU7qBX!HDNM;>!wXtre0OE zRAqdtxu(G$_J*317@iR_dCIO33zO5_OX3`ew~e&xp?4n(q%|qz#pByIZHFkw`PV(g zZq`&PZ@(`NWqdboqIuuis!1Uqb^XAXC|0nWM0&g5VL`|nh@Ya)*{-gw4W`+4NSp~@ zTCkZeCVgvI*w4?eOifGcC+oM#7tchgaAmfp09vTHv3C-zGcvcY1h>!s9~_iH-k|oS znm_10k!OuLB#Ge6N>5KO_u3j6H(~^8&T2@hdp_QP_j;Y+0sL+S(_5C7e`!R}EJW@P z`C3~LEn?lTJk*nn+2lcND>xx-JhLv=ZNtHvihR}|`-`*#K*G8_726HFRzj2PzR=b1 zy+AD&ByB2wbBV_LB2CIiiE)U(;ZNT0{-<&asLbS{lEH5eCzIknlbl)HaKqM)3Ukx) zLg#Y4-!R1^KR+Mpe2x7G#fM~%X-TXzD$?MuFh#>sE`t;{JmtJUhTDT>mr8hzLKtKZ zd3yF!D_+FXZ^ZcZjLRR_4gJQUQDAML5Vk@|;V!Lqfzw4;coc^}_1)+m9epwDI0Vmx zO5vwGy1gERX1r6qbV*w~fx0Wpq~2fW`6$AQiX_%)C_m}9Jt$2^ofF@nRaxG+QD@%? zGs9mFe+lq8vuQ?KH6Y&Z}7-sadCH^+6FI@nJPB@Xb zr%xf7&^U#mgsHof#0}?=A|^w4un(?)$ z>_e3z5;wS@5`ZJcaPD|iGUT=bO#0`qvQAg$6GG8XEa&2mGyGOtL5!FEr`?kB#;TVF zByoU>>|xNff98kNNh#{VLh3Q(*>P-C0}u`{)EsQKK%~a4S>iaMjHm+XBdI*){806fGStp}=QLB`df*#d6bXkaTG z2*Q{Ug&As8+~T+P_Hp3&?)=#f8Fw0{`%zKMaD_2$K0tl)-qvD2!2;u#QOdj}_?`YB z1bU&o$Rd^!M@eMl8W9l@vH8I`;!h`@5%>CFn~~OyL>Gn2ttRdy^P9JCxdjB67(7=H z4-&rbCG-_1===`OZE%KtZorEZ zqY`dAqXUfsuRl1fSY$+SnShzs07q~qkYdqh_$w1&AM!VmmxyVk(=T=zg!fK!HZ(Rh?ADfmMDg){PkK1${{Hbd)(5d~E@Cz>WSG&G3z++Q8x2XZGC z#eq==M1{`fZXj0lhOH$98xhfdI5)@0&;rk`1uM<~&L{rMX2>ZBVKgu^l>+y{S2mkj zNIIQ2$@8Az>3kA|^XfB2`US{lhu+B4aWek!wdR*xIBFRV)20xHJK9T9guXt%=#JP` z)QM@uLS$Tga6N?OwQOv}mfW82__}T^UsY>yt~ZdFuz=3)fXBqx@u~ZcU?EsGZSn&0 zWzS<(g@p}~m3V1D8O4A*yTWGF*SmnZ8_7Q}kV!)Lqpkv%Y&on@_hDXu{u<_ZWRoGJ zJ<8p^0Y0CHP&eY}%8*+n0K&Ii8UXPG&wZI;pt#1hhkGcU-gu3FE%E@dXI43ZsfBBR zw2WrHdWGTZe&l)=;zuIzDc4(?ITwCD3WeMN6Lh*R1 zCJM!Ry9EX^e@)j7MC>Ge4=`H@V-9Z~%+X|i5$S=z-W0OzDq1%tBMjLazvmtMNe%Hi z2{b;V1DAy#_WtmGckn08C%Zw=$EN-p@Paq}@6VGWM>6;GzkmLZ_pAQ* zH~oLmEBpWd?*AM8$!9zTz(y0p3gScX2(iAg9lfy=mL_$h1e@?U4#>;P7r{a8g`_26qriEp zT>_>$QD51r8%*GG5ygmE$7ONexFheV!_*)Z(yTROn;ChFq%KLdE?G&tMh^ymiW?FS zX+oxHQo?H@_;GqrESLYhc}_ocUe|deqWFQiTno5(pmk?mYg&UCupvm%=wg&gTGCND ztWLvcGLn`wlA|kCxoL9y9>= ztH1Z`PW^$)=E%*TY_YMi@sbC?chE!k})$RCt&8k5%R`& zdH2q`)3}%xg^e!~6ekZ4vA)qUFc3y*EenfkYK9KxJ)m}$h3y*lpsv9|;!wadO;lSl zhHQZb8xEMEJuu-wDnYII+qZ4R`-Vn0Pb5_N;eIrjV#!Kzzv_daAZjpypm+5!O2h8^ zgOfHnjze`kDBfAWh)pt*Ygp0#vkFlNh|0Jwo}LIQ(b3U20#6>CE`7+OG_^F>)bV=8 zb7fTn2QcqJ5J0ENeOvfQq`MvdNz=@2yW7Up9Tj76k*?Xdw;Mro!m)D*>_Y$?iR|Gn zJbU-91}(~p8rFAi&JHf#p<5D8bC&4t=q8tYUcS5&{z$X&o(jMPPb64F=&q!j{*>qK zi^2W2=!+4B@0uB`$_@xtAbP+DZu4U1AYjtH?Vxa(-<8x2RU3HO!xjla2VDHVaCylO zU3@THM~2@I8nptjVRsi-}y|MV<}1xykQ_ zP|5~2?fLVm(_aKHy>SCP;5u+%9UR%jM2U?iWY)Ytf532XR_kkT74|$hAUcXeeV=++ zgH&tbYlxCNgoQW2;UEieM;aeNDv&LOsRqTs#}A+c!bpW~NH(Hzf)IZ8y6T6|MGD|x zDb9{1I>CnQ;(c3CrK(@m9!YtFWxEFpcNZ`s9yuK&V<~V%EsX!raJxcvU5Y_N-;an0 zAmRfK&`J2_l?y^z&uVQGvs1mwz&$dX-T!JQ#%h56T|ptPOstw52D0$-<=WiO&)Z*#+z@>@d8W4rP5I(TnSw;8hs0MVFhd42sDdEBD4I z2sj~_(6THIS%(bY0Qb^^>hFl%I46oie>iJU5*5J*iZE3Hxmw#~<5uD{bZ`&_vF8P^HOC?Go0FR&Ck$08vO)5rOcw^5QU~>R23+KA1sNA#3d*?X253WaqST~f1j9U&f zqUC{R;ah?87M%Ptz`y%pVGKZ-C@e^u0zu}UUl*bDT21uG#wR2k$L2*xHcSN`I(?@v zO1K}2QetlvN*34Fzqrg>d#@7ERv(oilqe8sE74RF$f{#kJZN78Ho$7;=S38MB`{fI zE&ytU+Gm=n{)M_IXEXoWv>jZQeqY7%BXSfWzCbE_V>iJ~;4ti&%1;kh=@pXUI4? zD7C`sY(g|VikBf}3X&YQ3Ta}-IGl79Y1zq>1?Wgm88FdEQJ;ey%o3&`a=5T8Xb?JS zV8BhfB2eooE9$}6-s`%&z=0ak6P1J>DseKs2vX6+P*Riu`~ZVZ=}{dLm<4l>97f!0 z2nPTgY1l^EAeea$z7?CL3j&pPdf>QJI3hPm(cj2sJ-HyW( zz=EQm)K4g*Nn13rnx&EMQMB!G3kj_SsF#ECAEWV%@WrI0B6S@)A?9bt0`d8|goPWr z%4$#-V91N+E)&&_PzL?b=7suoZhoFTWSZssf=q_#3pkoD(OFH$Rg-Id6}IPY>`knZ zpi20W+bEFEDF*v1CcMXl1u{$mOv)z6N1%s_kPZAHD(6%8qpbjbn|-jWk|Ko27|>7f z0qBDj6K>tc_0kDPV1OV_itXYKMc5v&0}%clJ|LxF)!O8HhP5O{!}Gwob0;;@&I!ob zDHXHOy2un4NMH|jLP0uk;R_I?FREqh5Q(>-cgrI!O@I&+m>z{5oXupDkqA`9TDT(KP8GDmE#xxSO$#n^ z2i6jXdGtl@14gle`;8nMny3mwK_GIHS{4(uj#&3Il341$Y%x|{5_V+rh@0yp%SU&) z*HZ8Gp^aAE80WU?bn2TIwC66azIM&|=V;DCmd0~US%zfsyqC7(saemo)4w`&ojI+p zIzgenvufqWwHsI6d7yQVdWJdj3D0K=J-z0;!Y`7F2R`Q64@AsH*mvIQ(6v7*0T6V! z-jEJ*ucfW6=cRoW*wZX5EEEcHJfUR9>0-`N*&`qzi(UcH5?@VL^0|!~9pUBm3ZSn~ z(-KivSLXz$L;A!ZobCq#rnn^}@LZ`5_l2GXKoGFTP$r8N)&^4Iq9Dhp7QeS|MIS$X zy3*Lx74B@za-ji(BP=Y;DJaN{T_S~UEUhIb{XcUP)<7kS5P>Pz#Xzr`z~&oH+JHMtc#|r z~lUZ9u@6=YXpHY3l~I z2fonT_~c?vb1cAHugZVx`z)~H$O}rctHO@_o`f-?g7jv4dC?HKMbed&6a~@b%kN9* za>0f1l{X7k(bDb^6(wC(1fV*~#A-atuQOhn0Fz9~36wI11jZY{j*pBC8xZHubvt7c84rDWF zn)`Wb61#Mq#8?mk=sVk=6vut&Q1GO15#k`r`t|PUzcqmUdYfXPgrcG%A@vY$WQS*` zk0!#^LYmN_NLCfhfZpLm*hJaE9&rv(?v8X!5xj$*NRt6$djti&pFG)2DXOTTUH<*W z)e^}NT>9~y>oVGmu%y7epvldrMGl|W((-%waNXmuupfEhb#VM9O}x&{J=)T`1Ta`Cb^wvVuKd#x5u9LaGOJRDH4Xfr5AV~m3P;DHHPk-^E9C`rhU4XOe&cWWS*jVYLEQ!FkyCIxH z9+5+ovQgYpBiwrnuprz?;nb-zM5~r3-%&Ebk?eubNJwmu;4eETzTN8VhQ!zW>MHZn z(rn?iY+du=rx>M2wkJXpX^N^oKD3&Ffg-gdzZG815@Icz21ww6U*ayjO4ssqw6y~e zN$@hX@cXf^G?E$!xfpGtDgbPb5LO1L-|TP&3I0NX(@`90f=jFGz3}pc!0bRL*Gx9N zNCc>n??5lZe(=#VVs<0hO)u>yuY@15S^>mievp}#iJck;0FN7u{l2$;$0GQ55ui|? z#Ee3z*})q9TZA|SGe>4fK+o!MZfI=O2fzgeBzqa8UuEJnq6(TJZ=c{kuC5+bUcVcF z^F4Aw1_Cd?PTu3me1r>ls=t0%z;uTlTwGLW3q?(G?P=jR7BWpQ_tJ}royXzAd7hS* zf$fH^U}}~=k}Y?#ZLRjF{WVjNbpYtES&kI41v1EyLXiG%P*k8?G?+}xKG0n2_>f>$j0)2GA_5m97L^-&45F=m zAO;>t+t>*6?cF$jHZ%J?DwCkrjb?!T2M@|vTD~l2EG4iQW-`xM0BHmDu!e#vuJeb5*(lsiWC-11i;m}wGnB6WelIZX zYy^qmjf|(eF8XPgfM8w{K@(us(mop90v!D~xWU2sG_WaCxnbUy6!!zp9XnPceTa?) zet{9;nNT#}MeBHP{?xLm^{k@L<(-~mOZc}k?F;p1D=Mm{1D z0i8gE5#5w*4m8 zkNWIg=W7>wygujKi2SklH93qc|1??1$;lCiCS=~PP3Pt@H_6nt<(%({_fJDnkbCCl z+COW0yscQw^C9MW_4F7}POTJSVq((NT{+0NXAeEJdc+}Gl2zbp2VFryL#5`aM65@b zzMP{(#}3Bp z3?-VwN#jvc$JLNlrp+d2xWUmNFsr2m-D1hLk7S!)g2&@SrC;RVjEs36 zxACsI0B{hIlGD-}?F#M3tNZ7!SEH*CmC#B!)KXLVE@*1PwLvRY|MObG^xW#ztEq&m zZd@PEu!o6(fCwCOG6Lz=EpL>;78Oq+9;v)aKv-8r>;`(7ET1qQMNq_V@WgkFwRQrK zL4e;6f#}HC^hjH%NE$*&a%$=iRfnm@4o1ZSl3blCW%r8|S53OLRDj#a*6ehB??G>- z%yH$4DI(BAK&!5yNxA7dHd(;-Xx>oyJ znu`U#m5;KmYx@-1KV6E)L^aN#n{j~=rwoqFEodGOPnw!D*6PgLZQeJ8Y-9PeS%~}# zvT7eA_e8c*#KkL7;*MUZWq_I|a|AvM!Hm0o038{-s%M2JL>*>+e!Ms;4~-o2It&xD zB=8PCcnArDtZf8~Y&BZIm zsD#q0Ypxn_^YHTapcF`c@#0Eo=?{NljAPhNlslA7EE_g>Af&<_l2LD0ilONXxh8Eivt*1=iT5oIxQHsPTWovtgUqGPcNfhQRS{xKWDl;z_>+ADG zqyGj>Q>{S0ZMnEiZ1!u`eBIM8t)#q5w$iP`>2WiCnt5ejHLGDv_L|KP)Pd7UQU5ATomkCA^!nv$%!W)30y~f6%-A z=E(&~kUv20-^HHPZz^zV%K5oD_F!gqbzsiDbCq+)q|UbVJN8^gCzMY>fGmiBr)YC5 zM{`*@D0>^t(E)_pd49uI~0`mv|^e+jE}`CZQh3gg-DStM_7m@o1dSba{~i}^X8c``>tbwY(!24RY6*A zvBBcN9++Jev;VboEcZDxd>Sx1(86jB@_n+BrxC;^nDJi-((#(i#LDswUG<+aLEcS!j{LbpviO-+=PnNPg6091g?_UP=A?Bo`oh7%NmVLX7f$Sc3Zf&QqQyG#Z%1+14aBykO# z2BM2J=KuCaY+Ap4UF3jOniAOV5FxGOO*h8ex#2vQ{#6(@&{L>SWm z%aB=E=&18DxNXY#Crg7e`f=i6pQco0R)l*yRnl(eQ=wX~<&my}RUX+c3B?4gE6K+Q zO86_$Rf(<-Y|osmulTMs`G)`V^6z^V=gw`!k*75@G>mXByyk8}^T$u^!UZ~ng?A^; zIgUKcu@QLH-`@`tE4r*NhZdNOLJMr1>ljGMH*CVdapA*>J=`)F zu8t-}Iu%CdE1Y*y9CdhMOxg?Wp1|m}JLJZ%8yXp{f^CNkeFP~Buwe&!(^_SZWdD3s zWu@Pw&b0vrN29X3x`(gRjE#*4Rh2n-1Atl%M!TY6vwEG8SI_Y9>a%ChB4$u`baZG6 zicE8F*tChzU*W|;AQU<#?;ah<%IzQdkh9H%Bir2lcH)6%&ENF{ktei1k13x%ZE>rO z)wT)^vfpRDY5qNLQG2bMm_*Unjm`vfml+VF@Xr411YRSZ3+!9A5O>pzEn3u}R3I)B zPBE`D_|c=1i@S0Y5P>?LJi-pAM6Y!`o}d;J9qpEX^jaVqQa$L`tDZmajm=v9a0OrR zUF~&cA3o55u(q4|x!GlYXji}p!|K(9F7r9TogaC(hDW?;*A=+Xa3(q?dO&BE8T^j$2s?!6hji>81CH=GBSqy`a=_Ivb9iDq3D) z#v$Q!!7%L?VgeM3z*o*dS)-kWU+nW?W<@Gx@ja1Oxjwt!zDf2iZ1}3re%Co&$&HHQ zIKjOZ(m_?qJMz&8XeDR!z_G}SJ$EfsXY3!z?@WIF{JTw;&Kt{OJ*wxn0X8oQ^YV`uf!XP?yrGf+iplGEV94 zT?WivGG*Kij$dl$I*ded>0->GoHB15OFNjEk+Bwoe5Fu6QaE?-_DPEx9v-$omT``6 zqn1h-t-tGo$DCaP>79@;h%30`OV**^D~wGA!R#CvXo->W7XER;x&DHNh8J2FbOy%i zLy%tKLmr1mN)PpaL=K0{efgcBzZxEk)uUhl|NlgU@&^w{|N8ZNaGCsCb3nLod#zC$ zG(j1gwH-5_~`fc z3aov<7G#uM)=Ga^egdi|YrX@Nn~-V4ZrIWp;ZhJ=^LtBE2Dfq zSPBpVWcE4A_5BA9ya(Q$K(~@K_X9Hi=VuOp4QO9iB%Wgrz3g|DYps>LE4=tX!vv25uV!~j{t6iAO z6r;}9q5|o61CsBhcQ;R^XJuVZHf)WAL7W9e1tD!Qp`@5b!uijf5XilkAO=+=0x8? z7afr(cib@)mMC|y^|>yzbqY%y#QEPU>9Q5{8u1ID$9H^g!VvV7eyPt|_IW3;g}15yZxX=K+2&0CTg5YSq}-NThoclRXf5;_DyUe|fjB zE*Lu3E?vwHz#66qjkki8CPFu?bmAa+34#aXxC+%e#wXpz_b$yh6|(bdS7CMl9m;0n z&qUGDyt(Y)mCu7gi`hmXY%mUjC;`oeVNai~&b_q8!eZ=Z)2k-07e<>XWV{(PX6!mW zSOPKbX1D3JYgQ1wf`Wtfu~*W{*@u*`T)6_pi38~dUP2EL4brh43SCUE@)^jz&{aV8)?XbAW5v2P@&5aHY}Hy zoB`%uHs!xHBeTIdp`MfF!z3QuDbbR!3sb!B0IjNPXk@y-tTJgMfn>jk2rCgF!CL5< znGx?SfvuwoS(wM@QB$M(TqmC$Otdt(L?YVF&pSW{?n2O?gR!>7T(|VlN|afh9E732 zzkJX$gBk*vMl?_Uu4V{xD1%!y`^K2U&^DFrgnkL zwz^^|Dk^FKwFIG2(Go0J`&--9$YvPqK-7HF z^{MmqBN8QOr3!HG_wU~?>emK&N(`<6Ppva9_=B4!ZG>Q3iTp=6GkA>1jwSQrbWCaw ziSzRE64?(B2La*vr?K9;*V^OY~%ylq2ybK<*?! z0_P6@E(c6JQNG1O4^86XQqRC{5aJkz$Pyz*u(@Nd#c93Q;;z1&_}_N6Z;X#jREQoe&+63-LSAQZt#F`_s$XX0!?0akc_*L}_2!9nOobbp}z zW3;ckW|~)A+zx~%Y9%G5j^WqPWzaU(1^S-IQGnlg&mYz#li~!SllUuM=;Slt&E#SJ z3^5S2+}@*&bwU$)(5YSs#l%OCt)q{PfZJIbk8$YSB$KxM0|F)>$8duE$CiaUSw_p@ zh{G1qDG(`Wv4wCSPPE>+#_#Xw$QsfPF!K!lQqL38%+i<1A<>1Tas{re5~q1l|d zc+nTz9y-U3kW7FdyaM@O?W(`@57&2Q6WDYBKgm!_LhozArA1tlq-;yAN#l TJb%+m!5;-#l@l+d_3r!+tS1%% literal 0 HcmV?d00001 diff --git a/docs/source/models/supported-models.md b/docs/source/models/supported-models.md index c06243a4c53a..0340660acf8f 100644 --- a/docs/source/models/supported-models.md +++ b/docs/source/models/supported-models.md @@ -21,6 +21,7 @@ The following is a table of supported models for the PyTorch backend: | `Gemma4AssistantForCausalLM` | Gemma 4 MTP assistant | `google/gemma-4-E2B-it-assistant`, `google/gemma-4-E4B-it-assistant`, `google/gemma-4-26B-A4B-it-assistant`, `google/gemma-4-31B-it-assistant` | | `Glm4MoeForCausalLM` | GLM-4.5, GLM-4.6, GLM-4.7 | `THUDM/GLM-4-100B-A10B` | | `GlmMoeDsaForCausalLM` | GLM-5, GLM-5.2, GLM-5.3 | `zai-org/GLM-5`, `zai-org/GLM-5.2`, `zai-org/GLM-5.3` | +| `Glm5NextForConditionalGeneration` [^21] | GLM-5.3-Flash | `zai-org/GLM-5.3-Flash` | | `GptOssForCausalLM` | GPT-OSS | `openai/gpt-oss-20b`, `openai/gpt-oss-120b` | | `KimiK25ForConditionalGeneration` | Kimi-K2.5 | `moonshotai/Kimi-K2.5` | | `KimiK3ForConditionalGeneration` [^15]| Kimi-K3 | `moonshotai/Kimi-K3` | @@ -95,6 +96,7 @@ statuses for the same architecture in the two matrices. [^17]: Kimi K3 has no MTP or EAGLE-3 head, and its DSpark checkpoints are not compatible with plain `DFlash`. [^18]: NGram and standalone Suffix Automaton (SA) use model-free drafting on the PyTorch backend, so they are not listed in individual entries. This does not imply universal end-to-end support: compatibility depends on each model's multi-token verification and cache-management paths and may be untested or explicitly restricted. [^19]: KV cache reuse for hybrid recurrent-attention models requires an explicit recurrent-state snapshot policy, such as `kv_cache_config.mamba_state_config.periodic_snapshot_interval`; the model default disables reuse when no snapshot policy is configured. +[^21]: Supports text, image and video inputs and one-model MTP with 1 to 5 draft tokens. Requires the GLM-specific Transformers revision documented in the deployment guide, whose `Glm5NextProcessor` performs the image and video preprocessing. Beam search and the FP8 KV cache are not supported; disaggregated serving requires the Python NIXL transceiver; KV cache block reuse requires `kv_cache_config.mamba_state_config.periodic_snapshot_interval`. See the [deployment guide](../deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md). # Encoder-Decoder Feature Support Matrix (PyTorch Backend) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/__init__.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/__init__.py new file mode 100644 index 000000000000..0dc30a53395f --- /dev/null +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/__init__.py @@ -0,0 +1,37 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""glm_kpool: the GLM-5.3-Flash pool-compressed sparse-MLA backend.""" + +from .backend import ( + INDEX_SENTINEL, + GlmKpoolSparseAttention, + latent_pool_rows, + paged_slot_indices, + positions_to_pool_rows, +) +from .cache_manager import Glm5NextCacheManager, Glm5NextMamba2Metadata +from .params import GlmKpoolBackendForwardArgs, GlmKpoolSparseParams + +__all__ = [ + "INDEX_SENTINEL", + "Glm5NextCacheManager", + "Glm5NextMamba2Metadata", + "GlmKpoolBackendForwardArgs", + "GlmKpoolSparseAttention", + "GlmKpoolSparseParams", + "latent_pool_rows", + "paged_slot_indices", + "positions_to_pool_rows", +] diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py new file mode 100644 index 000000000000..fb63748c497d --- /dev/null +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py @@ -0,0 +1,891 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""GLM-5.3-Flash k-pool sparse MLA: the fully-NoPE branch of the TRTLLM family. + +``glm5_next`` sparse layers are fully NoPE MLA (``qk_rope_head_dim == 0``, +``qk_nope_head_dim == 256``, ``kv_lora_rank == 512``) whose visible set is +selected by a *pool-compressed* indexer: size-``index_kpool`` pools of keys are +scored, the top ``index_topk / index_kpool`` pools expand back into member +positions, and the incomplete tail is always appended, padded with ``-1`` to a +fixed logical width. Stock DeepSeek-V3.2 DSA selects individual keys and +assumes the 576-wide rope'd DeepSeek geometry (its SM100 absorption route +applies RoPE through ``mla_rope_append_paged_kv_assign_q``, and the shared +``MLA`` module's DSA hook requires the fused ``[q_a|kv_a|k_pe]`` projection and +``qk_rope_head_dim > 0``), so it cannot express this contract without inventing +a fake rotary width. + +:class:`GlmKpoolSparseAttention` is therefore a narrow **subclass of +:class:`~..trtllm.TrtllmAttention`** -- a sibling of +:class:`~.dsa.backend.DSATrtllmAttention` inside the TRTLLM sparse family -- +rather than a fork outside it: + +* **Family** -- it inherits the TRTLLM backend's identity wholesale: + ``support_mla()`` is true, ``is_mla_enable`` is true through a fully-NoPE + :class:`~..interface.MLAParams`, and ``Metadata`` is + :class:`~..trtllm.TrtllmAttentionMetadata`, the typed metadata class the + engine constructs from ``attn_backend.Metadata``. +* **Selection** -- ``get_attention_backend("TRTLLM", sparse_params)`` resolves + ``SparseParams(algorithm="glm_kpool")`` to this class next to the DSA branch + in the sparse registry, and the standard ``create_attention(...)`` dispatch + constructs it. +* **Contract** -- :meth:`forward` keeps the exact + ``AttentionBackend.forward(q, k, v, metadata, forward_args)`` signature: + options merge through :func:`~..interface.merge_attention_forward_args` + (which rejects unknown or mixed arguments), the model layer's pool-expanded + selection arrives through the typed + ``AttentionForwardArgs.sparse_backend_args.topk_indices`` carrier, and the + phase is declared through ``forward_args.attention_input_type``. +* **Cache ownership** -- every paged read and write is derived from the + prepared ``metadata``: the one hybrid ``KVCacheManagerV2`` reached through + ``metadata.kv_cache_manager`` owns the latent pages and the packed indexer + state, and the per-request block tables / visible lengths come from the + ``prepare()``-refreshed ``mamba_metadata.glm_*`` buffers. Callers never hand + raw pool tensors to this backend. +* **Execution** -- the absorbed sparse-MLA core dispatches + ``tensorrt_llm.flash_mla.flash_mla_sparse_fwd``, the FlashMLA sparse kernel + of the DSA stack (``sparse/dsa/module.py``), which natively supports + ``d_qk == d_v == 512`` and 64 query heads (``HEAD_DIM_512`` / + ``Fwd_Sm100_Head64_Impl`` in FlashMLA's ``sparse_fwd.h``) with the same + ``-1``/out-of-range invalid-index contract this model's indexer emits. + +The model layer (``modeling_glm5_next.Glm5NextSparseAttention``) keeps the +module math -- projections, norms, the pool indexer's scoring/selection +(model-layer sparse prediction, as in MiniMax-M3), query absorption, and the +output projection -- and drives this backend only through the standard +contract entry points. +""" + +from __future__ import annotations + +from collections.abc import Callable +from dataclasses import dataclass + +import torch + +from ...interface import ( + AttentionForwardArgs, + AttentionInputType, + MLAParams, + PositionalEmbeddingParams, + merge_attention_forward_args, +) +from ...trtllm import TrtllmAttention, TrtllmAttentionMetadata +from .kernels import kpool_expand, kpool_score, kpool_update +from .params import INDEX_SENTINEL, GlmKpoolSparseParams + + +def _flash_mla_sparse_fwd() -> Callable[..., tuple[torch.Tensor, torch.Tensor, torch.Tensor]]: + """The production sparse-MLA kernel entry point (lazy import). + + Resolved at call time so a test can intercept the module attribute to + prove the backend is the one dispatching it. + """ + try: + from tensorrt_llm.flash_mla import flash_mla_sparse_fwd + except ImportError as exc: # pragma: no cover - wheel always bundles it + raise RuntimeError( + "glm_kpool sparse MLA requires tensorrt_llm.flash_mla." + "flash_mla_sparse_fwd, which this build does not provide" + ) from exc + return flash_mla_sparse_fwd + + +def paged_slot_indices( + block_table: torch.Tensor, + positions: torch.Tensor, + tokens_per_block: int, +) -> tuple[torch.Tensor, torch.Tensor]: + """``(page, within_page)`` indices for ``positions`` in a paged pool. + + ``block_table`` is ``[..., max_pages]`` and ``positions`` is broadcastable + against its leading dimensions. The pair addresses a pool shaped + ``[num_pages, tokens_per_block, ...]``. + + Returning the pair rather than one flat ``page * tokens_per_block + offset`` + index is required, not stylistic. ``KVCacheManagerV2`` coalesces buffers + whose per-page size differs from K/V's into a shared pool, so the + ``Role.INDEX_KEY`` view it hands back is **strided**: its page stride is the + slot stride, not its own payload size. Measured on this model the indexer + buffer is ``[16, 64, 1, 256]`` with page stride 32768 against a 16384-element + payload. A flat index is only correct for a densely packed pool, and + flattening that view is not merely wrong -- ``.view`` raises and ``.reshape`` + silently *copies*, so every cache write would land in a temporary and be + lost. The pair is also what the accessor's own contract prescribes. + + Callers must mask invalid positions themselves -- this deliberately does not + invent a fallback page, because a silently wrong page is exactly the + cross-request leak the hybrid cache has to rule out. + """ + page = torch.div(positions, tokens_per_block, rounding_mode="floor") + offset = positions - page * tokens_per_block + return torch.gather(block_table, -1, page), offset + + +def latent_pool_rows(latent_pool: torch.Tensor) -> tuple[torch.Tensor, int, int]: + """Row-space view of the latent pool's storage for the sparse kernel. + + ``latent_pool`` is the slot-major ``[slots, tokens_per_block, dim]`` view + from ``Glm5NextCacheManager.get_latent_state_buffer``. V2 coalesces + several buffers into one pool, so the slot stride is larger than the + payload and the pool cannot be flattened with ``view`` (it would raise) + or ``reshape`` (it would silently copy every step). The kernel, however, + only needs *uniformly strided rows*: it reads ``kv_ptr + idx * stride``. + Every latent row already sits at a multiple of ``dim`` elements inside + the storage, so the whole storage is reinterpreted as ``[N, 1, dim]`` + rows and positions are translated to row ids instead. + + Returns ``(rows, base_row, rows_per_slot)`` where the row id of cache + position ``(slot, t)`` is ``base_row + slot * rows_per_slot + t``. Rows + belonging to other coalesced buffers are addressable but never indexed: + only ids produced by that formula (or ``-1`` sentinels) reach the kernel. + Pure metadata work -- no device kernel -- so it is CUDA-graph safe. + """ + slots, tokens_per_block, dim = latent_pool.shape + del slots, tokens_per_block + if latent_pool.stride(2) != 1 or latent_pool.stride(1) != dim: + raise ValueError( + "glm_kpool latent pool rows must be contiguous within a page; got " + f"strides {tuple(latent_pool.stride())} for dim {dim}" + ) + slot_stride = latent_pool.stride(0) + offset = latent_pool.storage_offset() + if slot_stride % dim or offset % dim: + raise ValueError( + f"glm_kpool latent pool slot stride {slot_stride} / storage offset " + f"{offset} are not multiples of dim {dim}; the pool has no uniform " + "row view and the coalesced layout assumption broke" + ) + total_rows = latent_pool.untyped_storage().nbytes() // (dim * latent_pool.element_size()) + rows = torch.as_strided(latent_pool, (total_rows, 1, dim), (dim, dim, 1), storage_offset=0) + return rows, offset // dim, slot_stride // dim + + +def positions_to_pool_rows( + positions: torch.Tensor, + block_table: torch.Tensor, + tokens_per_block: int, + base_row: int, + rows_per_slot: int, +) -> torch.Tensor: + """Request-local cache positions -> global row ids for the sparse kernel. + + ``positions`` is int32 ``[..., width]`` with :data:`INDEX_SENTINEL` in + invalid slots; ``block_table`` holds base slot ids per request. Sentinels + are preserved as ``-1`` (the kernel's own invalid marker) rather than + clamped -- a clamped sentinel would address a real row. Fixed shapes, + gathers, and ``where`` only, so a captured decode graph replays this + against prepare()-refreshed tables. + """ + safe = positions.clamp(min=0).long() + page = torch.div(safe, tokens_per_block, rounding_mode="floor") + slot = torch.gather(block_table, -1, page) + rows = base_row + slot * rows_per_slot + (safe - page * tokens_per_block) + return torch.where(positions >= 0, rows, positions.long()).to(torch.int32) + + +@dataclass(frozen=True) +class _GlmKpoolCacheState: + """One layer's cache state, derived from prepared attention metadata. + + ``latent_pool``/``index_pool`` are the slot-major ``[slots, + tokens_per_block, dim]`` views over the hybrid manager's coalesced pools; + ``block_tables``/``kv_lens`` cover the whole batch in executor order + (contexts first). Everything here is a view or a host int -- deriving it + launches no kernel, so it is safe inside a CUDA-graph capture as long as + the underlying buffers are the persistent ``prepare()``-refreshed ones. + """ + + latent_pool: torch.Tensor + index_pool: torch.Tensor + block_tables: torch.Tensor + kv_lens: torch.Tensor + tokens_per_block: int + num_contexts: int + + +class GlmKpoolSparseAttention(TrtllmAttention): + """Fully-NoPE k-pool sparse-MLA branch of the TRTLLM attention family. + + See the module docstring for the family/contract rationale. Constructed + under the standard ``create_attention(...)`` dispatch when + ``SparseParams(algorithm="glm_kpool")`` is configured on the TRTLLM + backend slot; the sparse registry resolves it next to + ``DSATrtllmAttention``. + """ + + Metadata = TrtllmAttentionMetadata + + #: FlashMLA tiles the top-k axis in blocks of 64 (``B_TOPK``); index rows + #: are padded to that multiple with ``-1`` (invalid) entries, which is + #: semantics-free by the kernel's own contract. + _KERNEL_TOPK_ALIGN = 64 + + #: Query-head counts the FlashMLA sparse kernel instantiates + #: (``Fwd_Sm100_Head64_Impl``/``Head128`` in ``sparse_fwd``; any other + #: ``h_q`` raises ``Unsupported h_q``). Under tensor parallelism the local + #: head count (16 of 64 at TP4) is below the smallest instantiation, so + #: :meth:`_dispatch_sparse_core` zero-pads the query-head axis up to the + #: next instantiated count and slices the output back -- the in-tree DSA + #: precedent (``sparse/dsa/module.py`` pads its TP-local heads the same + #: way). Attention is per-head, so zero query lanes cannot perturb real + #: lanes; their outputs are discarded by the slice. + _KERNEL_HEAD_COUNTS = (64, 128) + + def __init__( + self, + layer_idx: int, + num_heads: int, + head_dim: int, + num_kv_heads: int | None = None, + quant_config=None, + q_scaling: float | None = None, + pos_embd_params: PositionalEmbeddingParams | None = None, + mla_params: MLAParams | None = None, + skip_create_weights_in_init: bool = False, + attention_chunk_size: int | None = None, + sparse_params: GlmKpoolSparseParams | None = None, + dtype: torch.dtype | None = None, + aux_stream: torch.cuda.Stream | None = None, + **kwargs, + ) -> None: + # dtype/aux_stream arrive from the standard create_attention kwargs; + # this branch has no dtype-dependent weights and no side stream. + del dtype, aux_stream + # The engine-level construction path passes the llmapi config object; + # keep it (as DSA does) without forwarding it into the base class. + self.sparse_attention_config = kwargs.pop("sparse_attention_config", None) + if sparse_params is None: + raise ValueError("sparse_params is required for GlmKpoolSparseAttention") + if not isinstance(sparse_params, GlmKpoolSparseParams): + raise TypeError( + f"GlmKpoolSparseAttention needs GlmKpoolSparseParams, got {type(sparse_params)}" + ) + if head_dim != sparse_params.kv_lora_rank: + raise ValueError( + "glm_kpool consumes absorbed latent-space queries: head_dim " + f"({head_dim}) must equal kv_lora_rank ({sparse_params.kv_lora_rank})" + ) + if pos_embd_params is not None: + raise ValueError( + "glm_kpool is fully NoPE; positional embedding parameters have no " + "meaning on this branch" + ) + if mla_params is None: + # The standard create_attention MLA path asserts qk_rope_head_dim>0 + # (the rope'd DeepSeek geometry), so this fully-NoPE branch states + # its MLA identity itself instead of loosening the shared assert. + mla_params = MLAParams( + q_lora_rank=sparse_params.q_lora_rank, + kv_lora_rank=sparse_params.kv_lora_rank, + qk_rope_head_dim=0, + qk_nope_head_dim=sparse_params.qk_nope_head_dim, + v_head_dim=sparse_params.v_head_dim, + rope_append=False, + ) + if mla_params.qk_rope_head_dim != 0: + raise ValueError( + f"glm_kpool is fully NoPE; got qk_rope_head_dim={mla_params.qk_rope_head_dim}" + ) + TrtllmAttention.__init__( + self, + layer_idx, + num_heads, + head_dim, + num_kv_heads=num_kv_heads, + quant_config=quant_config, + q_scaling=q_scaling, + pos_embd_params=None, + mla_params=mla_params, + skip_create_weights_in_init=skip_create_weights_in_init, + attention_chunk_size=attention_chunk_size, + sparse_params=sparse_params, + **kwargs, + ) + if self.num_kv_heads != 1: + raise ValueError( + f"glm_kpool latent cache is MQA-style (one KV head), got {self.num_kv_heads}" + ) + #: Softmax scale of the *unabsorbed* q . k product over + #: ``qk_nope_head_dim``; absorption reassociates the matmuls but the + #: score scale is unchanged. + self.softmax_scale = float(sparse_params.qk_nope_head_dim) ** -0.5 + + @classmethod + def support_fused_rope(cls) -> bool: + # Fully NoPE: there is no rotary embedding anywhere on this path. + return False + + @classmethod + def support_fused_qkv(cls) -> bool: + # The model layer owns the low-rank q/kv projections and absorption. + return False + + # -- metadata-derived cache state ---------------------------------------- + + def _cache_state(self, metadata) -> _GlmKpoolCacheState: + """Derive this layer's cache state from the prepared metadata. + + The single source of cache truth for every entry point below. The + production path is the persistent one: ``prepare()`` with the + ``Glm5NextCacheManager`` attached refreshes the ``mamba_metadata``'s + ``glm_block_tables``/``glm_kv_lens`` device buffers, so a captured + decode graph replays against fresh values at stable addresses. + Harness metadata whose manager does not attach those buffers falls + back to an eager host-side derivation, which is refused under CUDA + graphs exactly like the model's runtime-context builder used to. + """ + if metadata is None: + raise ValueError( + "GlmKpoolSparseAttention requires the engine's prepared attention " + "metadata (TrtllmAttentionMetadata); got None. The backend derives " + "its cache pools, block tables, and visible lengths from it." + ) + manager = getattr(metadata, "kv_cache_manager", None) + if manager is None: + raise ValueError( + "glm_kpool metadata has no kv_cache_manager; the hybrid " + "KVCacheManagerV2 owns the latent/indexer pools" + ) + mamba_metadata = getattr(metadata, "mamba_metadata", None) + if mamba_metadata is None or mamba_metadata is False: + raise ValueError( + "glm_kpool requires prepared metadata: call metadata.prepare() " + "with the Glm5NextCacheManager attached (mamba_metadata is missing)" + ) + latent = manager.get_latent_state_buffer(self.layer_idx) + index = manager.get_index_state_buffer(self.layer_idx) + if latent is None or index is None: + raise ValueError( + f"glm_kpool layer {self.layer_idx} has no latent/indexer pool on " + "this manager; the layer schedule and the cache layout disagree" + ) + latent = latent[:, :, 0, :] + index = index[:, :, 0, :] + tokens_per_block = int(manager.tokens_per_block) + batch = int(metadata.seq_lens.shape[0]) + num_contexts = int(metadata.num_contexts) + + tables = getattr(mamba_metadata, "glm_block_tables", None) + if tables is not None: + # Persistent path: slices of prepare()-refreshed buffers. No + # allocation, no H2D, no host sync -- CUDA-graph safe. Visible + # lengths prefer the metadata's device-corrected kv_lens_cuda + # (overlap scheduler + speculative decoding rewinds it in-graph; + # see modeling_glm5_next.glm5_next_visible_lens). + # Kept in the metadata's own integer width: every consumer here + # (comparisons, the fused kernels) accepts int32 or int64, and a + # per-call cast would launch one kernel per entry point per layer. + live = getattr(metadata, "kv_lens_cuda", None) + kv_lens = live[:batch] if live is not None else mamba_metadata.glm_kv_lens[:batch] + return _GlmKpoolCacheState( + latent_pool=latent, + index_pool=index, + block_tables=tables[:batch], + kv_lens=kv_lens, + tokens_per_block=tokens_per_block, + num_contexts=num_contexts, + ) + + # Legacy eager derivation for harness managers that do not attach the + # GLM buffers. It allocates and copies, so it must never run inside a + # captured region. + if getattr(metadata, "is_cuda_graph", False): + raise RuntimeError( + "glm_kpool CUDA-graph execution requires the persistent " + "prepare()-refreshed glm_block_tables/glm_kv_lens buffers; the " + "attached mamba_metadata has no glm_block_tables" + ) + kv_params = getattr(metadata, "kv_cache_params", None) + if kv_params is None or kv_params.num_cached_tokens_per_seq is None: + raise ValueError("glm_kpool requires kv_cache_params.num_cached_tokens_per_seq") + lens = [int(n) for n in metadata.seq_lens[:batch]] + cached = [int(n) for n in kv_params.num_cached_tokens_per_seq[:batch]] + device = latent.device + # Raw base-slot IDs, NOT get_batch_cache_indices: the latent/index + # views are slot-major, and V2's standard accessor scales page ids for + # its own flattened per-layer views. + pages = manager.get_batch_slot_tables(list(metadata.request_ids)[:batch]) + max_pages = max((len(p) for p in pages), default=1) or 1 + block_tables = torch.zeros(batch, max_pages, dtype=torch.long, device=device) + for row, page_ids in enumerate(pages): + if page_ids: + block_tables[row, : len(page_ids)] = torch.as_tensor( + page_ids, dtype=torch.long, device=device + ) + kv_lens = torch.as_tensor( + [c + n for c, n in zip(cached, lens)], dtype=torch.long, device=device + ) + return _GlmKpoolCacheState( + latent_pool=latent, + index_pool=index, + block_tables=block_tables, + kv_lens=kv_lens, + tokens_per_block=tokens_per_block, + num_contexts=num_contexts, + ) + + # -- paged cache path ----------------------------------------------------- + + def append_paged_state( + self, + latent: torch.Tensor, + packed: torch.Tensor, + positions: torch.Tensor, + metadata, + *, + request_index: int | None = None, + request_ids: torch.Tensor | None = None, + ) -> None: + """Write new tokens' latent and packed indexer state to the pools. + + ``positions`` carries the tokens' cache positions (schedule, owned by + the model layer); which pools and tables they land in is derived from + ``metadata``. ``request_index`` selects one context request's table + row; ``request_ids`` (``[tokens]`` int32, executor request index per + packed context token) addresses all context requests at once; + ``None`` for both addresses the generation rows, with ``positions`` + shaped ``[num_generations, 1]``. Callers pass only positions they own + -- see :func:`paged_slot_indices` for why no fallback page is invented. + """ + state = self._cache_state(metadata) + if request_ids is not None: + # Packed context rows of several requests: row i's page comes from + # block table request_ids[i] (no [tokens, max_pages] gather). + page_idx = torch.div(positions, state.tokens_per_block, rounding_mode="floor") + offset = positions - page_idx * state.tokens_per_block + page = state.block_tables[request_ids.long(), page_idx] + else: + if request_index is None: + table = state.block_tables[state.num_contexts :] + else: + table = state.block_tables[request_index] + page, offset = paged_slot_indices(table, positions, state.tokens_per_block) + state.latent_pool[page, offset] = latent.to(state.latent_pool.dtype) + # Only the [k | gate] columns; the pool-key slice is maintained by + # update_pool_keys. + packed_dim = self.sparse_params.packed_state_dim + state.index_pool[page, offset, :packed_dim] = packed.to(state.index_pool.dtype) + + def gather_paged_prefix( + self, + length: int, + metadata, + *, + request_index: int, + ) -> tuple[torch.Tensor, torch.Tensor]: + """One context request's cached latent and packed-indexer prefix (host + loop path: prefill only, never captured).""" + state = self._cache_state(metadata) + positions = torch.arange(length, device=state.latent_pool.device) + page, offset = paged_slot_indices( + state.block_tables[request_index], positions, state.tokens_per_block + ) + packed = state.index_pool[page, offset][..., : self.sparse_params.packed_state_dim] + return state.latent_pool[page, offset], packed + + def _rows( + self, + state: _GlmKpoolCacheState, + request_index: int | None, + num_rows: int, + rows_per_request: int = 1, + request_ids: torch.Tensor | None = None, + ) -> tuple[torch.Tensor, torch.Tensor | None]: + """``(block_tables, kv_lens)`` for the rows the fast-path kernels see. + + ``None`` addresses the generation rows: one row per generation request + (visible length from the metadata), or ``rows_per_request`` rows per + request -- a speculative verification pass -- whose block tables are + repeated per row and whose per-row visible lengths the caller + supplies. An ``int`` addresses one context request: its single block + table is broadcast (stride 0) over the request's ``num_rows`` query + tokens, whose per-row visible lengths the caller supplies (each query + sees its own prefix). ``request_ids`` addresses the packed rows of all + context requests: the kernels index the batch's block tables through + it (row ``i`` -> table ``request_ids[i]``), so nothing is gathered. + """ + if request_ids is not None: + return state.block_tables, None + if request_index is None: + gen = slice(state.num_contexts, None) + if rows_per_request > 1: + return state.block_tables[gen].repeat_interleave(rows_per_request, dim=0), None + return state.block_tables[gen], state.kv_lens[gen] + table = state.block_tables[request_index].unsqueeze(0).expand(num_rows, -1) + return table, None + + def update_pool_keys( + self, + positions: torch.Tensor, + ape: torch.Tensor, + metadata, + *, + request_index: int | None = None, + request_ids: torch.Tensor | None = None, + ) -> None: + """Refresh the pool containing each of ``positions``. + + Generation rows (``request_index=None``): ``positions`` is + ``[num_generations]``, the cache position each request just wrote. + One context request: ``positions`` are that request's pool-final + positions written this chunk (one per pool, so no two programs write + the same pool). ``ape`` is the indexer's ``[kpool, head_dim]`` + compress APE. All context requests at once: ``request_ids`` maps each + position to its request. One fused kernel, in place, CUDA-graph safe. + """ + state = self._cache_state(metadata) + tables, _ = self._rows(state, request_index, positions.shape[0], request_ids=request_ids) + kpool_update( + state.index_pool, + tables, + positions, + ape, + state.tokens_per_block, + head_dim=self.sparse_params.index_head_dim, + kpool=self.sparse_params.index_kpool, + request_ids=request_ids, + ) + + def score_pools( + self, + q: torch.Tensor, + weights: torch.Tensor, + metadata, + *, + q_scale: float, + w_scale: float, + request_index: int | None = None, + kv_lens: torch.Tensor | None = None, + rows_per_request: int = 1, + request_ids: torch.Tensor | None = None, + ) -> torch.Tensor: + """Fused pool scoring -> ``[N, P_cap]`` fp32. + + ``q`` is ``[N, n_heads, head_dim]``, ``weights`` ``[N, n_heads]``. + Reads the cached pool keys directly (work proportional to each row's + visible length, not to the buffer capacity); pools that are not yet + complete hold the fp32 minimum. Generation rows by default; for one + context request pass ``request_index`` and the per-query-token + visible lengths ``kv_lens`` (``position + 1``); for a speculative + verification pass over the generation rows pass ``rows_per_request`` + (tokens per request) and the per-row ``kv_lens``. + """ + state = self._cache_state(metadata) + tables, gen_lens = self._rows( + state, request_index, q.shape[0], rows_per_request, request_ids=request_ids + ) + capacity = state.block_tables.shape[1] * state.tokens_per_block + kpool = self.sparse_params.index_kpool + return kpool_score( + q, + weights, + state.index_pool, + tables, + gen_lens if kv_lens is None else kv_lens, + state.tokens_per_block, + num_pools_max=(capacity + kpool - 1) // kpool, + head_dim=self.sparse_params.index_head_dim, + kpool=self.sparse_params.index_kpool, + q_scale=q_scale, + w_scale=w_scale, + # bf16 inputs are exact in tf32, so the tensor-core dot is an fp32 + # accumulation of exact products (measured: identical selections). + precision="tf32", + # Context rows: the query tokens of a request share its block + # table, so a program gathers each pool-key block once for 16 rows. + rows_per_program=1 if request_index is None and request_ids is None else 16, + request_ids=request_ids, + ) + + def expand_selection( + self, + selected: torch.Tensor, + metadata, + *, + request_index: int | None = None, + kv_lens: torch.Tensor | None = None, + rows_per_request: int = 1, + request_ids: torch.Tensor | None = None, + ) -> torch.Tensor: + """Selected pools ``[N, select_k]`` -> latent row ids for the kernel. + + Expands each valid pool into its members, appends the always-visible + tail, pads with ``-1`` to :attr:`GlmKpoolSparseParams.kernel_output_width` + and translates positions to latent-cache row ids in one kernel. Same + row addressing as :meth:`score_pools`. + """ + state = self._cache_state(metadata) + tables, gen_lens = self._rows( + state, request_index, selected.shape[0], rows_per_request, request_ids=request_ids + ) + _, base_row, rows_per_slot = latent_pool_rows(state.latent_pool) + return kpool_expand( + selected, + gen_lens if kv_lens is None else kv_lens, + tables, + state.tokens_per_block, + base_row=base_row, + rows_per_slot=rows_per_slot, + kpool=self.sparse_params.index_kpool, + out_width=self.sparse_params.kernel_output_width, + request_ids=request_ids, + ) + + # -- sparse core ---------------------------------------------------------- + + def create_output( + self, + q: torch.Tensor, + *, + is_quantize_output: bool, + metadata=None, + attention_mask=None, + is_gen_only: bool = False, + **kwargs, + ) -> list[torch.Tensor]: + """Allocate the standard flat output buffer for this absorbed branch. + + Reconciles the inherited ``TrtllmAttention.create_output``, whose MLA + context leg allocates ``num_heads * v_head_dim``: the absorbed + formulation emits the *latent* width ``num_heads * kv_lora_rank`` in + **both** phases (the model layer applies the absorbed V projection to + it afterwards, so ``v_head_dim`` never appears at this boundary). + Quantized/NVFP4 output modes are not implemented on this branch and + are rejected loudly rather than silently mis-allocated. + """ + del metadata, attention_mask, is_gen_only, kwargs + if is_quantize_output: + raise ValueError( + "glm_kpool produces a bf16 latent-space output; quantized " + "attention output (out_scale/output_sf) is not supported" + ) + return [q.new_empty((q.shape[0], self.num_heads * self.kv_lora_rank), dtype=q.dtype)] + + def _dispatch_sparse_core( + self, + q_latent: torch.Tensor, + kv_rows: torch.Tensor, + topk_rows: torch.Tensor, + ) -> torch.Tensor: + """Pad the index rows to the kernel's tiles and run the sparse kernel. + + ``q_latent`` is ``[T, H, kv_lora]``, ``kv_rows`` is ``[N, 1, + kv_lora]``, ``topk_rows`` is int32 ``[T, width]`` with + :data:`INDEX_SENTINEL` invalid slots. Returns the kernel's latent-space + output ``[T, H, kv_lora]``; :meth:`_finalize_output` flattens it to the + base-contract shape at the ``forward`` boundary. + """ + pad = (-topk_rows.shape[-1]) % self._KERNEL_TOPK_ALIGN + if pad: + topk_rows = torch.nn.functional.pad(topk_rows, (0, pad), value=INDEX_SENTINEL) + local_heads = q_latent.shape[1] + kernel_heads = next((h for h in self._KERNEL_HEAD_COUNTS if h >= local_heads), None) + if kernel_heads is None: + raise ValueError( + f"glm_kpool: {local_heads} query heads exceed every FlashMLA " + f"sparse-kernel instantiation {self._KERNEL_HEAD_COUNTS}" + ) + if kernel_heads != local_heads: + # Zero-filled query lanes; per-head attention keeps them inert and + # the slice below discards their outputs (DSA's TP-head padding). + q_padded = q_latent.new_zeros((q_latent.shape[0], kernel_heads, q_latent.shape[2])) + q_padded[:, :local_heads, :] = q_latent + q_latent = q_padded + out, _, _ = _flash_mla_sparse_fwd()( + q_latent, + kv_rows, + topk_rows.unsqueeze(1), + self.softmax_scale, + self.kv_lora_rank, + ) + if kernel_heads != local_heads: + # Strided view over the real heads; the model's absorbed V bmm + # consumes strided batches, so no copy of [T, H, kv_lora] here. + out = out[:, :local_heads, :] + return out + + def _finalize_output( + self, out_latent: torch.Tensor, output: torch.Tensor | None + ) -> torch.Tensor: + """Flatten the kernel output to ``[T, num_heads * kv_lora]``. + + With no caller buffer the kernel's own (contiguous) allocation is + returned viewed flat -- same shape/dtype/device as + :meth:`create_output` would allocate, without a redundant copy. A + caller-provided ``forward_args.output`` (already validated in + :meth:`forward`) is written in place and returned, matching the + TRTLLM family's caller-owned-buffer semantics. + """ + if output is None: + # The real heads remain contiguous within each token even when + # the token stride includes padded heads. Flattening these two + # axes keeps a view and preserves the backend's 2-D contract. + return out_latent.flatten(1) + output.copy_(out_latent.reshape(out_latent.shape[0], -1)) + return output + + def forward( + self, + q: torch.Tensor, + k: torch.Tensor | None, + v: torch.Tensor | None, + metadata: TrtllmAttentionMetadata, + forward_args: AttentionForwardArgs | None = None, + **kwargs, + ) -> torch.Tensor: + """Standard ``AttentionBackend.forward`` for the k-pool sparse core. + + Arguments follow the base contract exactly: + + ``q`` + Absorbed latent-space queries, ``[T, num_heads * kv_lora]`` or the + equivalent ``[T, num_heads, kv_lora]`` view. + ``k`` + Context phase: the request's contiguous cached latent prefix + ``[KV, kv_lora]`` (``num_kv_heads == 1``), gathered from the same + metadata through :meth:`gather_paged_prefix`. Generation phase: + ``None`` -- the kernel reads the paged latent pool directly + through the storage row view derived from ``metadata``. + ``v`` + Must be ``None``: the latent rows serve as both K and V, exactly + as in absorbed MLA; the model layer applies the absorbed V + projection to the returned latent output. + ``metadata`` + The engine's prepared typed metadata. Required: the paged pools, + block tables, and visible lengths are derived from it (see + :meth:`_cache_state`); ``None`` or unprepared metadata is a loud + error. + ``forward_args`` / ``**kwargs`` + Merged through :func:`merge_attention_forward_args`, which + rejects unknown kwargs and forward_args/kwargs mixing. The + model layer's pool-expanded selection travels in the typed + ``sparse_backend_args.topk_indices`` field (int32 + ``[T, output_width]``, request-local positions, + :data:`INDEX_SENTINEL` padding), and + ``attention_input_type`` declares the phase -- this backend is + phase-explicit, so ``mixed`` is rejected. A caller-provided + ``forward_args.output`` is validated (shape/dtype/device), + written in place, and returned -- the TRTLLM family's + caller-owned-buffer semantics. Quantized-output modes + (``out_scale``/``out_scale_sf``/``output_sf``) are not + implemented here and are rejected loudly. + + Returns the flat latent-space attention output + ``[T, num_heads * kv_lora]`` -- the base contract's + ``(num_q_tokens, num_heads * head_dim)`` with this backend's + ``head_dim == kv_lora_rank``. The model layer views it per-head for + the absorbed V projection. + """ + forward_args = merge_attention_forward_args(forward_args, kwargs) + sparse_args = forward_args.sparse_backend_args + topk_indices = sparse_args.topk_indices if sparse_args is not None else None + topk_rows = getattr(sparse_args, "topk_rows", None) + if topk_indices is None and topk_rows is None: + raise NotImplementedError( + "GlmKpoolSparseAttention needs the model layer's pool-expanded " + "selection in forward_args.sparse_backend_args.topk_indices; the " + "k-pool scoring/selection is module math -- see " + "modeling_glm5_next.Glm5NextSparseAttention." + ) + if v is not None: + raise ValueError("glm_kpool consumes latent rows as both K and V; v must be None") + if q.dim() == 2: + q = q.view(q.shape[0], self.num_heads, self.head_dim) + + if ( + forward_args.out_scale is not None + or forward_args.out_scale_sf is not None + or forward_args.output_sf is not None + ): + raise ValueError( + "glm_kpool does not support quantized attention output " + "(out_scale/out_scale_sf/output_sf); it returns the bf16 " + "latent-space output the model layer projects" + ) + output = forward_args.output + if output is not None: + expected_shape = (q.shape[0], self.num_heads * self.kv_lora_rank) + if ( + tuple(output.shape) != expected_shape + or output.dtype != q.dtype + or output.device != q.device + ): + raise ValueError( + f"glm_kpool forward_args.output must be a {expected_shape} " + f"tensor of dtype {q.dtype} on {q.device}; got shape " + f"{tuple(output.shape)}, dtype {output.dtype}, device " + f"{output.device}" + ) + + state = self._cache_state(metadata) + input_type = forward_args.attention_input_type + if topk_rows is not None: + # Fast path (either phase): the selection was expanded and + # translated to latent-pool row ids by expand_selection, so the + # kernel reads the paged pool directly; k must not be passed. + if k is not None: + raise ValueError("glm_kpool: topk_rows addresses the paged pool; k must be None") + if input_type not in ( + AttentionInputType.context_only, + AttentionInputType.generation_only, + ): + raise ValueError(f"glm_kpool forward is phase-explicit, got {input_type!r}") + kv_rows, _, _ = latent_pool_rows(state.latent_pool) + return self._finalize_output(self._dispatch_sparse_core(q, kv_rows, topk_rows), output) + if input_type == AttentionInputType.context_only: + if k is None: + raise ValueError( + "glm_kpool context forward needs the request's contiguous " + "latent prefix as k (see gather_paged_prefix)" + ) + if topk_indices is None: + raise ValueError( + "glm_kpool context forward selects over the request's own " + "contiguous prefix; pass request-local topk_indices, not rows" + ) + kv_len = k.shape[0] + out_latent = self._dispatch_sparse_core( + q, k.view(kv_len, 1, self.kv_lora_rank), topk_indices + ) + return self._finalize_output(out_latent, output) + if input_type == AttentionInputType.generation_only: + if k is not None: + raise ValueError( + "glm_kpool generation forward reads the paged latent pool " + "from metadata; k must be None" + ) + gen_tables = state.block_tables[state.num_contexts :] + num_gens = gen_tables.shape[0] + if num_gens == 0 or q.shape[0] % num_gens: + raise ValueError( + f"glm_kpool generation forward got {q.shape[0]} query rows for " + f"{num_gens} generation requests in the metadata" + ) + kv_rows, base_row, rows_per_slot = latent_pool_rows(state.latent_pool) + # Speculative verification packs ``1 + runtime_draft_len`` query + # rows per generation request, request-major; every row of a + # request resolves its selection through that request's table. + tokens_per_request = q.shape[0] // num_gens + if tokens_per_request > 1: + gen_tables = gen_tables.repeat_interleave(tokens_per_request, dim=0) + topk_rows = positions_to_pool_rows( + topk_indices, gen_tables, state.tokens_per_block, base_row, rows_per_slot + ) + return self._finalize_output(self._dispatch_sparse_core(q, kv_rows, topk_rows), output) + raise ValueError( + "glm_kpool forward is phase-explicit: set " + "forward_args.attention_input_type to context_only or " + f"generation_only, got {input_type!r}" + ) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py new file mode 100644 index 000000000000..8a025ddcf7fd --- /dev/null +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py @@ -0,0 +1,303 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Cache manager and prepared metadata of the glm_kpool sparse backend. + +``Glm5NextMamba2Metadata`` is the model's mamba metadata: the shared Kimi KDA +metadata (chunk schedule, aligned generation slot indices for the replay +verify kernel) plus the sparse layers' persistent buffers below. + +CUDA graphs replay captured kernels over fixed buffer addresses; nothing +Python re-runs at replay. Every per-step request-derived value the sparse +layers consume in decode must therefore live in persistent device buffers +refreshed by ``prepare()`` -- which the engine (and the CUDA-graph runner, +before every replay) calls outside the captured region. This subclass adds +exactly those buffers to the mamba metadata the hybrid manager already +attaches: + +* ``glm_block_tables`` -- ``[max_batch, max_blocks_per_seq]`` base-slot + page ids (pinned staging + one async H2D per step); +* ``glm_kv_lens`` -- ``[max_batch]`` per-request visible lengths + (cached + this step's tokens), same staging pattern; +* ``glm_cached_lens_host`` / ``glm_ctx_cu_seqlens`` -- plain host values + for the prefill path, which is never captured (decode-only graphs). + +The buffer width is fixed at first use from the manager's own +``max_blocks_per_seq`` so captured gathers can never need a wider table; +a mid-run widening request is a hard error rather than a silent +reallocation that stale graphs would keep reading. +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import torch + +from tensorrt_llm._torch.modules.kimi_kda.kimi_k3_mamba_metadata import KimiK3MambaMetadata +from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import Role +from tensorrt_llm._torch.pyexecutor.kv_cache.mamba_cache_manager import MambaHybridCacheManagerV2 +from tensorrt_llm.runtime.kv_cache_manager_v2 import BufferConfig + + +class Glm5NextMamba2Metadata(KimiK3MambaMetadata): + def __init__( + self, max_batch_size: int, chunk_size: int, max_num_tokens: int | None = None + ) -> None: + # The KDA chunk-index buffer holds one row per 64-token chunk; a + # harness that gives no token budget gets a generous fixed capacity + # (two int64 per row -- negligible). + super().__init__( + max_batch_size, chunk_size, max_num_tokens if max_num_tokens is not None else 65536 + ) + from tensorrt_llm._utils import prefer_pinned + + self._glm_pin = prefer_pinned() + self.glm_block_tables: torch.Tensor | None = None + self._glm_block_tables_cpu: torch.Tensor | None = None + self.glm_kv_lens = torch.zeros(max_batch_size, dtype=torch.long, device="cuda") + self._glm_kv_lens_cpu = torch.zeros( + max_batch_size, dtype=torch.long, pin_memory=self._glm_pin + ) + self.glm_cached_lens_host: list[int] = [] + self.glm_ctx_cu_seqlens: list[int] = [0] + + def _glm_ensure_tables(self, width: int) -> None: + width = max(1, int(width)) + if self.glm_block_tables is None: + self.glm_block_tables = torch.zeros( + self.max_batch_size, width, dtype=torch.long, device="cuda" + ) + self._glm_block_tables_cpu = torch.zeros( + self.max_batch_size, width, dtype=torch.long, pin_memory=self._glm_pin + ) + elif self.glm_block_tables.shape[1] < width: + raise RuntimeError( + "glm5_next block-table buffer would need to grow from " + f"{self.glm_block_tables.shape[1]} to {width} pages mid-run; " + "captured CUDA graphs would keep reading the old buffer" + ) + + def prepare(self, attn_metadata) -> None: + super().prepare(attn_metadata) + manager = attn_metadata.kv_cache_manager + kv_params = attn_metadata.kv_cache_params + request_ids = attn_metadata.request_ids + if ( + manager is None + or not hasattr(manager, "get_batch_slot_tables") + or kv_params is None + or kv_params.num_cached_tokens_per_seq is None + or request_ids is None + ): + return + + batch = attn_metadata.seq_lens.shape[0] + num_contexts = int(attn_metadata.num_contexts) + lens = [int(x) for x in attn_metadata.seq_lens[:batch]] + cached_src = kv_params.num_cached_tokens_per_seq + if isinstance(cached_src, torch.Tensor): + cached = [int(x) for x in cached_src[:batch]] + else: + cached = [int(cached_src[i]) for i in range(batch)] + + self.glm_cached_lens_host = cached + cu = [0] + for length in lens[:num_contexts]: + cu.append(cu[-1] + length) + self.glm_ctx_cu_seqlens = cu + + self._glm_kv_lens_cpu[:batch].copy_( + torch.as_tensor([c + n for c, n in zip(cached, lens)], dtype=torch.long) + ) + self.glm_kv_lens[:batch].copy_(self._glm_kv_lens_cpu[:batch], non_blocking=True) + + width = int(getattr(manager, "max_blocks_per_seq", 0)) or 1 + self._glm_ensure_tables(width) + pages = manager.get_batch_slot_tables(list(request_ids)[:batch]) + staging = self._glm_block_tables_cpu + staging[:batch].zero_() + for row, page_ids in enumerate(pages): + if page_ids: + staging[row, : len(page_ids)].copy_(torch.as_tensor(page_ids, dtype=torch.long)) + self.glm_block_tables[:batch].copy_(staging[:batch], non_blocking=True) + + +class Glm5NextCacheManager(MambaHybridCacheManagerV2): + """One KVCacheManagerV2 lifecycle for all three kinds of GLM state. + + The three state families have genuinely different shapes and are kept + that way -- none is padded into a faux common KV tensor: + + * linear-attention layers: the recurrent accumulator and the four-tap + convolution history, carried by the inherited Mamba side with + ``conv_state_layout='q_k_v'`` (the convolution's ``[q | k | v]`` + section order); + * sparse-attention layers: the ``kv_lora_rank``-wide compressed latent, + carried by the standard attention pages with ``SELFKONLY``; + * sparse-attention layers again: the indexer's packed ``[k | gate]`` + state, added here as one extra ``Role.INDEX_KEY`` buffer per sparse + layer. + + The extra buffer is registered through the base class's own + ``_extra_buffers_per_layer`` hook, so allocation, block reuse, slot + release, and disaggregated bookkeeping keep working. A second manager + would duplicate request ownership and break exactly those lifecycles. + """ + + def __init__( + self, *args, sparse_layer_ids: Sequence[int] = (), index_state_dim: int = 0, **kwargs + ) -> None: + # Set before super().__init__: the base _build_base_config calls + # _extra_buffers_per_layer, which reads both of these. + self.sparse_layer_ids = sorted(int(i) for i in sparse_layer_ids) + self.index_state_dim = int(index_state_dim) + if self.sparse_layer_ids and self.index_state_dim <= 0: + raise ValueError( + "glm5_next sparse layers need a positive index_state_dim " + f"(got {self.index_state_dim})" + ) + super().__init__(*args, **kwargs) + + def _extra_buffers_per_layer(self, *, tokens_per_block: int) -> dict[int, list[BufferConfig]]: + """One ``Role.INDEX_KEY`` buffer per sparse layer, keyed by local id.""" + elem_bytes = torch.tensor([], dtype=torch.bfloat16).element_size() + size_per_block = self.index_state_dim * elem_bytes * tokens_per_block + return { + self.layer_offsets[layer_id]: [BufferConfig(role=Role.INDEX_KEY, size=size_per_block)] + for layer_id in self.sparse_layer_ids + if layer_id in self.layer_offsets + } + + def get_index_state_buffer(self, layer_idx: int) -> torch.Tensor | None: + """Paged indexer state for ``layer_idx``, NHD-shaped.""" + return self.get_index_k_buffer( + layer_idx, + num_heads=1, + head_dim=self.index_state_dim, + dtype=torch.bfloat16, + kv_layout="NHD", + ) + + def _sparse_pool_id(self) -> int: + """The single V2 layer-group id that owns every sparse layer. + + Base page indices are per layer group, so one block table can + address both the latent and the indexer views only because all + sparse layers -- whose KEY and INDEX_KEY buffers share each + layer's group -- resolve to one group. Asserted, not assumed: + a future geometry change that splits the group must fail here + rather than silently interleave two slot spaces. + """ + pools = { + self.layer_to_pool_mapping_dict[self.layer_offsets[layer_id]] + for layer_id in self.sparse_layer_ids + if layer_id in self.layer_offsets + } + if len(pools) != 1: + raise ValueError( + f"glm5_next sparse layers span V2 layer groups {sorted(pools)}; " + "the slot-indexed latent/index views require a single group" + ) + return pools.pop() + + def get_batch_slot_tables(self, request_ids: Sequence[int]) -> list[list[int]]: + """Raw base-slot IDs per request, for the slot-major state views. + + ``get_batch_cache_indices`` is scaled for V2's flattened + per-layer page views -- it returns ``base * scale // kv_factor`` + (scale is the coalesced buffers-per-slot count; 11 on the real + checkpoint) -- so feeding its output to the slot-major views from + :meth:`get_latent_state_buffer` / :meth:`get_index_state_buffer` + addresses the wrong slots and eventually runs past the pool. + Those views are indexed by the *base* page id itself, so this + accessor requests the identity conversion from the same V2 + bookkeeping (``is_kv_aggregate=False, index_scale=1``). + + A pipeline-parallel rank whose local slice holds no sparse layer + (e.g. the first three GLM layers are all linear attention) has no + sparse pool at all; it gets empty rows, which no layer on that + rank ever reads. + """ + if not any(layer_id in self.layer_offsets for layer_id in self.sparse_layer_ids): + return [[] for _ in request_ids] + return self._get_batch_cache_indices_by_pool_id( + list(request_ids), + pool_id=self._sparse_pool_id(), + is_kv_aggregate=False, + index_scale=1, + ) + + def get_latent_state_buffer(self, layer_idx: int) -> torch.Tensor | None: + """Paged latent state for ``layer_idx``, addressed by **slot** id. + + ``get_buffers`` hands back a view whose dim-0 stride is a *single* + page, but V2 coalesces every layer's ``Role.KEY`` buffer into one + pool and starts layer ``L``'s view ``L`` pages into it. Indexing two + layers' views with the same block id therefore makes them overlap + almost entirely -- measured on this checkpoint, eleven sparse layers + share a 2046-page pool and each layer's page ``p`` is the next + layer's page ``p - 1``. + + V2's own callers never index that view with a raw block id: + ``_get_batch_cache_indices_by_pool_id`` multiplies every base page + index by ``get_layer_page_index_scale(layer_idx)`` first. Folding + that scale into the *view* rather than into each caller's block + table is exactly what :meth:`get_index_k_buffer` already does for + ``Role.INDEX_KEY`` (``[slots, scale, ...][:, 0]``), and it is the + only convention under which one block table can address both pools: + the coalesced INDEX_KEY view is expressible *only* slot-indexed. + + Returns ``[num_slots, tokens_per_block, num_kv_heads, head_dim]``, + the same rank as :meth:`get_index_state_buffer`, so a caller + decomposes a position into one ``(slot, within_slot)`` pair and uses + it against both. + """ + pages = self.get_buffers(layer_idx) + if pages is None: + return None + flat = pages[:, 0] # [pages, tokens, heads, dim] + converter = self.impl.get_page_index_converter(self.layer_offsets[layer_idx], Role.KEY) + # ``scale`` is the buffers-per-slot count of the coalesced pool; + # ``within_slot`` is where this layer's buffer sits inside a slot. + scale, within_slot = int(converter.scale), int(converter.layer_offset) + kv_factor = int(self.kv_factor) + if scale <= kv_factor: + return flat + if scale % kv_factor: + raise ValueError( + f"glm5_next layer {layer_idx}: page-index scale {scale} is not a " + f"multiple of kv_factor {kv_factor}, so the latent pool has no " + "slot-major view" + ) + # ``get_buffers`` already folds the kv_factor axis in, so its dim 0 + # counts kv-aggregate pages and the slot stride is the scale in those + # units. Slot counts are derived the way get_index_k_buffer derives + # them, so both pools expose the same slot space. + stride = scale // kv_factor + slots = (flat.shape[0] + within_slot) // stride + # Slot s lives at pool page ``within_slot + s * stride``; the last one + # stays in range because a layer's offset inside a coalesced slot is + # always smaller than the slot itself. + if within_slot >= stride or (slots - 1) * stride >= flat.shape[0]: + raise ValueError( + f"glm5_next layer {layer_idx}: {slots} slots at stride {stride} do " + f"not fit {flat.shape[0]} pages from slot offset {within_slot}" + ) + return torch.as_strided( + flat, + size=(slots, *flat.shape[1:]), + stride=(stride * flat.stride(0), *flat.stride()[1:]), + storage_offset=flat.storage_offset(), + ) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py new file mode 100644 index 000000000000..bf32fac7b368 --- /dev/null +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py @@ -0,0 +1,427 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Triton kernels of the glm_kpool sparse backend: pool-key refresh, pool scoring and selection +expansion over the paged indexer cache. Fixed shapes, no host synchronization, CUDA-graph safe. +""" + +from __future__ import annotations + +import torch +import triton +import triton.language as tl + +_FP32_MIN = torch.finfo(torch.float32).min + + +@triton.jit +def _kpool_update_kernel( + POOL, + BT, + POS, + APE, + REQ, + slot_stride, + row_stride, + bt_stride, + tpb, + HD: tl.constexpr, + KPOOL: tl.constexpr, + HAS_REQ: tl.constexpr, +): + """Recompute the pool key of the pool containing ``POS[b]``. + + Members beyond ``POS[b]`` are masked (they are not visible yet), so an + incomplete pool carries the key of its visible members and is finalized by + the write of its last member. Row ``b`` reads block table ``REQ[b]`` when + ``HAS_REQ`` (several rows of one request share a table), else table ``b``. Numerics follow ``build_pools``: fp32 + softmax over ``gate + ape``, bf16 probabilities and products, fp32 sum, + bf16 result. + """ + b = tl.program_id(0) + if HAS_REQ: + tbl = tl.load(REQ + b).to(tl.int64) + else: + tbl = b.to(tl.int64) + pos = tl.load(POS + b).to(tl.int64) + start = (pos // KPOOL) * KPOOL + m = tl.arange(0, KPOOL) + d = tl.arange(0, HD) + mem_pos = start + m + valid = mem_pos <= pos + page = mem_pos // tpb + off = mem_pos - page * tpb + slot = tl.load(BT + tbl * bt_stride + page, mask=valid, other=0).to(tl.int64) + row = POOL + slot * slot_stride + off * row_stride + k = tl.load(row[:, None] + d[None, :], mask=valid[:, None], other=0.0).to(tl.float32) + g = tl.load(row[:, None] + HD + d[None, :], mask=valid[:, None], other=0.0).to(tl.float32) + ape = tl.load(APE + m[:, None] * HD + d[None, :]).to(tl.float32) + logits = tl.where(valid[:, None], g + ape, float("-inf")) + mx = tl.max(logits, axis=0) + e = tl.exp(logits - mx[None, :]) + probs = e / tl.sum(e, axis=0)[None, :] + probs = probs.to(tl.bfloat16).to(tl.float32) + prod = (probs * k).to(tl.bfloat16).to(tl.float32) + pool_key = tl.sum(prod, axis=0) + page0 = start // tpb + off0 = start - page0 * tpb + slot0 = tl.load(BT + tbl * bt_stride + page0).to(tl.int64) + tl.store(POOL + slot0 * slot_stride + off0 * row_stride + 2 * HD + d, pool_key.to(tl.bfloat16)) + + +def kpool_update( + index_pool: torch.Tensor, + block_tables: torch.Tensor, + positions: torch.Tensor, + ape: torch.Tensor, + tokens_per_block: int, + *, + head_dim: int, + kpool: int, + request_ids: torch.Tensor | None = None, +) -> None: + """Refresh the pool keys of the pools containing ``positions`` (one per row). + + ``index_pool`` is the slot-major ``[slots, tokens_per_block, 3 * head_dim]`` + view, ``block_tables`` ``[N, max_pages]`` int64, ``positions`` ``[N]``. + With ``request_ids`` (``[N]`` int32) row ``i`` reads block table + ``block_tables[request_ids[i]]`` instead of row ``i`` -- the packed rows of + several context requests. In-place; no host sync, fixed shapes: + CUDA-graph safe. + """ + n = positions.shape[0] + if n == 0: + return + _kpool_update_kernel[(n,)]( + index_pool, + block_tables, + positions, + ape, + positions if request_ids is None else request_ids, + index_pool.stride(0), + index_pool.stride(1), + block_tables.stride(0), + tokens_per_block, + HD=head_dim, + KPOOL=kpool, + HAS_REQ=request_ids is not None, + num_warps=4, + ) + + +@triton.jit +def _kpool_score_kernel( + Q, + W, + POOL, + BT, + KV_LENS, + REQ, + OUT, + num_rows, + slot_stride, + row_stride, + bt_stride, + tpb, + num_pools_max, + q_scale, + w_scale, + min_value, + HD: tl.constexpr, + KPOOL: tl.constexpr, + H: tl.constexpr, + HP: tl.constexpr, + BP: tl.constexpr, + ROWS: tl.constexpr, + PRECISION: tl.constexpr, + HAS_REQ: tl.constexpr, +): + """Score ``BP`` pools for ``ROWS`` consecutive rows: ``sum_h w[h] * relu(q_h . key_j * s)``. + + Pools whose last member is not yet visible to a row (``j >= kv_len // + KPOOL``) get ``min_value`` so a following top-k never picks them ahead of a + candidate. Row ``r`` reads block table ``REQ[r]`` when ``HAS_REQ`` (packed + context rows of several requests), else table ``r``. ``ROWS > 1`` is for + query tokens that share a block table (one request, or one request per + group under ``HAS_REQ``): the ``BP`` pool keys are gathered once and reused + for every row of the program. A group straddling two requests (only at + request boundaries) gathers per row instead. + """ + r0 = tl.program_id(0) * ROWS + j0 = tl.program_id(1) * BP + j = j0 + tl.arange(0, BP) + d = tl.arange(0, HD) + h = tl.arange(0, HP) + hmask = h < H + r_last = tl.minimum(r0 + ROWS - 1, num_rows - 1) + pos = j.to(tl.int64) * KPOOL + page = pos // tpb + off = pos - page * tpb + if HAS_REQ: + t0 = tl.load(REQ + r0).to(tl.int64) + t_last = tl.load(REQ + r_last).to(tl.int64) + shared = t0 == t_last + else: + t0 = r0.to(tl.int64) + shared = True + if shared: + # The last row of the group has the largest visible length (rows of + # one request are in position order); a block past its candidates is + # past every row's. + kv_last = tl.load(KV_LENS + r_last).to(tl.int64) + if j0 >= kv_last // KPOOL: + for i in tl.static_range(ROWS): + r = r0 + i + tl.store( + OUT + r * num_pools_max + j, + tl.full([BP], min_value, tl.float32), + mask=(j < num_pools_max) & (r < num_rows), + ) + else: + valid_any = j < kv_last // KPOOL + slot = tl.load(BT + t0 * bt_stride + page, mask=valid_any, other=0).to(tl.int64) + kptr = POOL + slot * slot_stride + off * row_stride + 2 * HD + keys = tl.load(kptr[:, None] + d[None, :], mask=valid_any[:, None], other=0.0).to( + tl.float32 + ) + for i in tl.static_range(ROWS): + r = r0 + i + in_range = r < num_rows + kv_len = tl.load(KV_LENS + r, mask=in_range, other=0).to(tl.int64) + valid = j < kv_len // KPOOL + q = tl.load( + Q + r * H * HD + h[:, None] * HD + d[None, :], + mask=hmask[:, None] & in_range, + other=0.0, + ).to(tl.float32) + scores = tl.dot(q, tl.trans(keys), input_precision=PRECISION) # [HP, BP] + scores = tl.maximum(scores * q_scale, 0.0) + w = tl.load(W + r * H + h, mask=hmask & in_range, other=0.0).to(tl.float32) + mixed = tl.sum(scores * (w * w_scale)[:, None], axis=0) + mixed = tl.where(valid, mixed, min_value) + tl.store(OUT + r * num_pools_max + j, mixed, mask=(j < num_pools_max) & in_range) + else: + for i in tl.static_range(ROWS): + r = r0 + i + in_range = r < num_rows + kv_len = tl.load(KV_LENS + r, mask=in_range, other=0).to(tl.int64) + tbl = tl.load(REQ + r, mask=in_range, other=0).to(tl.int64) + valid = j < kv_len // KPOOL + slot = tl.load(BT + tbl * bt_stride + page, mask=valid, other=0).to(tl.int64) + kptr = POOL + slot * slot_stride + off * row_stride + 2 * HD + keys = tl.load(kptr[:, None] + d[None, :], mask=valid[:, None], other=0.0).to( + tl.float32 + ) + q = tl.load( + Q + r * H * HD + h[:, None] * HD + d[None, :], + mask=hmask[:, None] & in_range, + other=0.0, + ).to(tl.float32) + scores = tl.dot(q, tl.trans(keys), input_precision=PRECISION) # [HP, BP] + scores = tl.maximum(scores * q_scale, 0.0) + w = tl.load(W + r * H + h, mask=hmask & in_range, other=0.0).to(tl.float32) + mixed = tl.sum(scores * (w * w_scale)[:, None], axis=0) + mixed = tl.where(valid, mixed, min_value) + tl.store(OUT + r * num_pools_max + j, mixed, mask=(j < num_pools_max) & in_range) + + +def kpool_score( + q: torch.Tensor, + weights: torch.Tensor, + index_pool: torch.Tensor, + block_tables: torch.Tensor, + kv_lens: torch.Tensor, + tokens_per_block: int, + *, + num_pools_max: int, + head_dim: int, + kpool: int, + q_scale: float, + w_scale: float, + precision: str = "ieee", + rows_per_program: int = 1, + request_ids: torch.Tensor | None = None, +) -> torch.Tensor: + """Pool scores ``[N, num_pools_max]`` fp32. + + ``q`` is ``[N, H, head_dim]`` (bf16 or fp32, contiguous), ``weights`` + ``[N, H]``, ``kv_lens[i]`` row ``i``'s visible length. Only the + ``kv_len // kpool`` complete pools of each row are scored; the rest hold + the fp32 minimum. ``rows_per_program > 1`` requires rows that share a + block table to be consecutive with non-decreasing ``kv_lens``: either all + rows (``block_tables`` broadcast with stride 0, one context request) or, + with ``request_ids`` (``[N]`` int32, row -> block-table row), the packed + query tokens of several context requests in position order. Fixed shapes, + no host sync. + """ + n, num_heads, hd = q.shape + if hd != head_dim: + raise ValueError(f"kpool_score: q head_dim {hd} != {head_dim}") + out = torch.empty(n, num_pools_max, dtype=torch.float32, device=q.device) + if n == 0: + return out + rows = max(1, int(rows_per_program)) + if rows > 1 and n > 1 and request_ids is None and block_tables.stride(0) != 0: + raise ValueError( + "kpool_score: rows_per_program > 1 needs a broadcast block table or request_ids" + ) + bp = 64 + grid = (triton.cdiv(n, rows), triton.cdiv(num_pools_max, bp)) + _kpool_score_kernel[grid]( + q, + weights, + index_pool, + block_tables, + kv_lens, + kv_lens if request_ids is None else request_ids, + out, + n, + index_pool.stride(0), + index_pool.stride(1), + block_tables.stride(0), + tokens_per_block, + num_pools_max, + q_scale, + w_scale, + _FP32_MIN, + HD=head_dim, + KPOOL=kpool, + H=num_heads, + HP=max(16, triton.next_power_of_2(num_heads)), + BP=bp, + ROWS=rows, + PRECISION=precision, + HAS_REQ=request_ids is not None, + num_warps=4, + ) + return out + + +@triton.jit +def _kpool_expand_kernel( + SEL, + KV_LENS, + BT, + REQ, + OUT, + sel_stride, + bt_stride, + tpb, + base_row, + rows_per_slot, + out_width, + SELECT_K: tl.constexpr, + KPOOL: tl.constexpr, + KPOOL_P2: tl.constexpr, + PAD_BLOCK: tl.constexpr, + HAS_REQ: tl.constexpr, +): + """Expand selected pools into latent-cache row ids and append the tail. + + Output row ``b`` is ``[SELECT_K * KPOOL expanded members | KPOOL - 1 tail + positions | -1 padding]``; every invalid entry is ``-1`` (the FlashMLA + sparse kernel's own invalid marker). Row id of cache position ``p`` is + ``base_row + block_table[p // tpb] * rows_per_slot + p % tpb``. + """ + b = tl.program_id(0) + if HAS_REQ: + tbl = tl.load(REQ + b).to(tl.int64) + else: + tbl = b.to(tl.int64) + kv_len = tl.load(KV_LENS + b).to(tl.int64) + num_cand = kv_len // KPOOL + + i = tl.arange(0, SELECT_K) + j = tl.load(SEL + b * sel_stride + i).to(tl.int64) + # A radix top-k pads short rows with negative ids; those are invalid too. + valid = (j >= 0) & (j < num_cand) + m = tl.arange(0, KPOOL_P2) + mmask = m < KPOOL + pos = j[:, None] * KPOOL + m[None, :] + page = pos // tpb + off = pos - page * tpb + ok = valid[:, None] & mmask[None, :] + slot = tl.load(BT + tbl * bt_stride + page, mask=ok, other=0).to(tl.int64) + rows = base_row + slot * rows_per_slot + off + rows = tl.where(ok, rows, -1) + tl.store( + OUT + b * out_width + i[:, None] * KPOOL + m[None, :], + rows.to(tl.int32), + mask=mmask[None, :] & (i[:, None] >= 0), + ) + + # Tail: the incomplete trailing group, always visible to its own query. + tail_count = kv_len - num_cand * KPOOL + tail_start = kv_len - tail_count + t = tl.arange(0, KPOOL_P2) + tpos = tail_start + t + tvalid = t < tail_count + tpage = tpos // tpb + toff = tpos - tpage * tpb + tslot = tl.load(BT + tbl * bt_stride + tpage, mask=tvalid, other=0).to(tl.int64) + trows = tl.where(tvalid, base_row + tslot * rows_per_slot + toff, -1) + tidx = SELECT_K * KPOOL + t + tl.store(OUT + b * out_width + tidx, trows.to(tl.int32), mask=tidx < out_width) + + # -1 padding up to the kernel-aligned width. + pad0 = SELECT_K * KPOOL + KPOOL_P2 + for start in range(pad0, out_width, PAD_BLOCK): + p = start + tl.arange(0, PAD_BLOCK) + tl.store(OUT + b * out_width + p, tl.full([PAD_BLOCK], -1, tl.int32), mask=p < out_width) + + +def kpool_expand( + selected: torch.Tensor, + kv_lens: torch.Tensor, + block_tables: torch.Tensor, + tokens_per_block: int, + *, + base_row: int, + rows_per_slot: int, + kpool: int, + out_width: int, + request_ids: torch.Tensor | None = None, +) -> torch.Tensor: + """Selected pools ``[N, select_k]`` -> latent row ids ``[N, out_width]`` int32. + + ``out_width`` must cover ``select_k * kpool + (kpool - 1)``; extra columns + are ``-1`` padding (FlashMLA tiles the top-k axis in 64s). ``request_ids`` + (``[N]`` int32) maps rows to block-table rows as in :func:`kpool_score`. + """ + n, select_k = selected.shape + if out_width < select_k * kpool + kpool - 1: + raise ValueError(f"kpool_expand: out_width {out_width} < {select_k * kpool + kpool - 1}") + out = torch.empty(n, out_width, dtype=torch.int32, device=selected.device) + if n == 0: + return out + _kpool_expand_kernel[(n,)]( + selected, + kv_lens, + block_tables, + kv_lens if request_ids is None else request_ids, + out, + selected.stride(0), + block_tables.stride(0), + tokens_per_block, + base_row, + rows_per_slot, + out_width, + SELECT_K=select_k, + KPOOL=kpool, + KPOOL_P2=triton.next_power_of_2(kpool), + PAD_BLOCK=64, + HAS_REQ=request_ids is not None, + num_warps=4, + ) + return out diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/params.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/params.py new file mode 100644 index 000000000000..febdf64a4910 --- /dev/null +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/params.py @@ -0,0 +1,115 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Lowered parameters and typed forward arguments of the glm_kpool sparse backend.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Literal + +import torch + +from ..params import SparseBackendForwardArgs, SparseParams + +#: Invalid-slot marker in every index tensor this backend consumes. It is the +#: FlashMLA sparse kernel's own invalid contract ("-1 or >= s_kv"), so padded +#: or unselected slots pass through unchanged; clamping them into range would +#: attend a real row. +INDEX_SENTINEL = -1 + + +@dataclass(frozen=True) +class GlmKpoolSparseParams(SparseParams): + """Lowered runtime parameters for the GLM k-pool sparse-MLA backend.""" + + algorithm: Literal["glm_kpool"] = field(init=False, default="glm_kpool") + #: Latent (compressed KV) width; also the absorbed query head width and + #: the kernel's d_qk == d_v. 512 on this checkpoint. + kv_lora_rank: int = 512 + #: Pre-absorption query/key head width; sets the softmax scale. Fully + #: NoPE: there is no rope component on top of it. + qk_nope_head_dim: int = 256 + #: Low-rank query bottleneck width; carried into ``MLAParams`` so the + #: backend's MLA identity states the real checkpoint geometry. + q_lora_rank: int = 1536 + #: Per-head value width after the absorbed V projection. + v_head_dim: int = 256 + #: Number of key positions the expanded selection may cover. + index_topk: int = 2048 + #: Members per compressed pool. + index_kpool: int = 4 + #: Whether the incomplete trailing pool is always appended. + index_always_select_tail: bool = True + #: Indexer key width; the cached per-token row is ``[k | gate | pool key]``. + index_head_dim: int = 128 + + def __post_init__(self) -> None: + if self.index_kpool <= 0 or self.index_kpool & (self.index_kpool - 1): + raise ValueError("glm_kpool requires a positive power-of-two index_kpool") + if self.index_topk <= 0 or self.index_topk % self.index_kpool: + raise ValueError( + "glm_kpool requires index_topk to be a positive multiple of index_kpool" + ) + if not self.index_always_select_tail: + raise ValueError("glm_kpool requires index_always_select_tail=True") + + @property + def indices_block_size(self) -> int: + return 1 + + @property + def packed_state_dim(self) -> int: + """Width of the per-token ``[k | gate]`` pair the model layer writes.""" + return 2 * self.index_head_dim + + @property + def cache_row_dim(self) -> int: + """Width of one indexer cache row: ``[k | gate | pool key]``. + + The pool key of pool ``j`` lives in the trailing ``index_head_dim`` + columns of the row at position ``j * index_kpool`` (its first member); + it is maintained incrementally by :meth:`GlmKpoolSparseAttention. + update_pool_keys` so decode never rebuilds pools from scratch. + """ + return 3 * self.index_head_dim + + @property + def select_k(self) -> int: + """Number of pools a query selects.""" + return self.index_topk // self.index_kpool + + @property + def kernel_output_width(self) -> int: + """``output_width`` padded to the FlashMLA top-k tile (64).""" + return -(-self.output_width // 64) * 64 + + @property + def output_width(self) -> int: + """Fixed logical width of the expanded index rows the model emits.""" + return self.index_topk + (self.index_kpool - 1 if self.index_always_select_tail else 0) + + +@dataclass(kw_only=True, slots=True) +class GlmKpoolBackendForwardArgs(SparseBackendForwardArgs): + """``SparseBackendForwardArgs`` plus the backend's own row-id selection. + + ``topk_rows`` carries a selection already translated to latent-cache row + ids (int32 ``[T, kernel_output_width]``, ``-1`` invalid) by + :meth:`GlmKpoolSparseAttention.expand_selection`; when present it is + consumed directly and ``topk_indices`` (request-local positions) is not + needed. + """ + + topk_rows: torch.Tensor | None = None diff --git a/tensorrt_llm/_torch/attention/backends/sparse/registry.py b/tensorrt_llm/_torch/attention/backends/sparse/registry.py index 5b0520e77392..a47c27e60ff6 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/registry.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/registry.py @@ -110,6 +110,15 @@ def get_trtllm_sparse_attn_attention_backend( # returns an instantiable AttentionBackend under the trtllm # attention backend slot. return _resolve_minimax_m3_backend_cls(sparse_params) + elif sparse_params.algorithm == "glm_kpool": + # GLM-5.3-Flash fully-NoPE pool-compressed sparse MLA: a narrow + # TrtllmAttention subclass beside the DSA branch (stock DSA assumes + # the 576-wide rope'd DeepSeek geometry). The model layer predicts the + # pool-expanded indices; the backend owns the metadata-derived paged + # latent/indexer cache path and the FlashMLA sparse-MLA core. + from .glm_kpool import GlmKpoolSparseAttention + + return GlmKpoolSparseAttention else: raise ValueError( f"Unsupported sparse attention algorithm in trtllm attention backend: {sparse_params.algorithm}" diff --git a/tensorrt_llm/_torch/model_config.py b/tensorrt_llm/_torch/model_config.py index 209a09e37a9b..d0e3d697383f 100644 --- a/tensorrt_llm/_torch/model_config.py +++ b/tensorrt_llm/_torch/model_config.py @@ -31,9 +31,10 @@ from tensorrt_llm._torch.locality_domain.policy import LocalityDomainPolicy from tensorrt_llm._torch.pyexecutor.config_utils import ( + get_glm5_next_layer_masks, get_glm5_next_num_attention_layers, get_kimi_linear_num_attention_layers, get_qwen3_hybrid_num_attention_layers, - is_kimi_linear, is_nemotron_hybrid, is_qwen3_hybrid, is_qwen4_exp, - load_pretrained_config) + is_glm5_next, is_kimi_linear, is_nemotron_hybrid, is_qwen3_hybrid, + is_qwen4_exp, load_pretrained_config) from tensorrt_llm._utils import (get_sm_version, is_sm_100f, torch_dtype_to_binding) from tensorrt_llm.bindings import LayerType as LayerTypeCpp @@ -1600,6 +1601,8 @@ def get_num_attention_layers(self) -> int: return get_qwen3_hybrid_num_attention_layers(cfg) if is_kimi_linear(cfg): return get_kimi_linear_num_attention_layers(cfg) + if is_glm5_next(cfg): + return get_glm5_next_num_attention_layers(cfg) return cfg.num_hidden_layers def get_num_mamba_layers(self) -> int: @@ -1613,6 +1616,11 @@ def get_num_mamba_layers(self) -> int: if is_kimi_linear(cfg): return cfg.num_hidden_layers - get_kimi_linear_num_attention_layers( cfg) + if is_glm5_next(cfg): + # From the literal layer_types list: the composite config may not + # carry num_hidden_layers at the top level before normalization. + _, kda_mask = get_glm5_next_layer_masks(cfg) + return sum(kda_mask) return 0 diff --git a/tensorrt_llm/_torch/models/__init__.py b/tensorrt_llm/_torch/models/__init__.py index 3fa0f8237940..607ea87c43b3 100644 --- a/tensorrt_llm/_torch/models/__init__.py +++ b/tensorrt_llm/_torch/models/__init__.py @@ -44,6 +44,8 @@ "Gemma4ForConditionalGeneration", "Gemma4UnifiedForConditionalGeneration", "Glm4MoeForCausalLM", + "Glm5NextForCausalLM", + "Glm5NextVLM", "GptOssForCausalLM", "HCXVisionForCausalLM", "HunYuanDenseV1ForCausalLM", diff --git a/tensorrt_llm/_torch/models/_arch_index.py b/tensorrt_llm/_torch/models/_arch_index.py index c1631932079f..7aeda02ae58a 100644 --- a/tensorrt_llm/_torch/models/_arch_index.py +++ b/tensorrt_llm/_torch/models/_arch_index.py @@ -58,6 +58,8 @@ def is_builtin_zoo_module(module_name: str) -> bool: "Gemma4ForConditionalGeneration": "modeling_gemma4mm", "Gemma4UnifiedForConditionalGeneration": "modeling_gemma4_unified", "Glm4MoeForCausalLM": "modeling_glm", + "Glm5NextForCausalLM": "modeling_glm5_next", + "Glm5NextForConditionalGeneration": "modeling_glm5_next_vision", "GlmMoeDsaForCausalLM": "modeling_deepseekv3", "GptOssForCausalLM": "modeling_gpt_oss", "HCXVisionForCausalLM": "modeling_hyperclovax", @@ -141,6 +143,8 @@ def is_builtin_zoo_module(module_name: str) -> bool: "Gemma4ForConditionalGeneration": "modeling_gemma4mm", "Gemma4UnifiedForConditionalGeneration": "modeling_gemma4_unified", "Glm4MoeForCausalLM": "modeling_glm", + "Glm5NextForCausalLM": "modeling_glm5_next", + "Glm5NextVLM": "modeling_glm5_next_vision", "GptOssForCausalLM": "modeling_gpt_oss", "HCXVisionForCausalLM": "modeling_hyperclovax", "HunYuanDenseV1ForCausalLM": "modeling_hunyuan_dense", @@ -202,6 +206,7 @@ def is_builtin_zoo_module(module_name: str) -> bool: # ``modeling_qwen_image_bench`` with byte-identical metadata; the real model's # module is indexed. MULTIMODAL_MODEL_TYPE_TO_MODULE = { + "glm5_next": "modeling_glm5_next_vision", "NemotronH_Nano_Omni_Reasoning_V3": "modeling_nemotron_h_multimodal", "NemotronH_Nano_VL_V2": "modeling_nemotron_h_multimodal", "cosmos3": "modeling_cosmos3", diff --git a/tensorrt_llm/_torch/models/checkpoints/__init__.py b/tensorrt_llm/_torch/models/checkpoints/__init__.py index beb03c998a11..47446452dd79 100644 --- a/tensorrt_llm/_torch/models/checkpoints/__init__.py +++ b/tensorrt_llm/_torch/models/checkpoints/__init__.py @@ -8,6 +8,7 @@ from .hf.cosmos3_weight_mapper import Cosmos3HfWeightMapper from .hf.gemma3_weight_mapper import Gemma3HfWeightMapper from .hf.gemma4_weight_mapper import Gemma4HfWeightMapper +from .hf.glm5_next_weight_mapper import Glm5NextHfWeightMapper from .hf.llama4_weight_mapper import Llama4HfWeightMapper from .hf.llava_next_weight_mapper import LlavaNextHfWeightMapper from .hf.mixtral_weight_mapper import MixtralHfWeightMapper @@ -40,5 +41,5 @@ "Gemma4HfWeightMapper", "LlavaNextHfWeightMapper", "MistralLarge3CheckpointLoader", "MistralLarge3WeightMapper", "MXCheckpointLoader", "Qwen3VLHfWeightMapper", "Cosmos3HfWeightMapper", - "Qwen4ExpHfWeightMapper" + "Qwen4ExpHfWeightMapper", "Glm5NextHfWeightMapper" ] diff --git a/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py b/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py new file mode 100644 index 000000000000..916b2e7a461d --- /dev/null +++ b/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py @@ -0,0 +1,265 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Checkpoint-key mapping for GLM-5.3-Flash (``glm5_next``). + +Everything here is a pure function of checkpoint key names and the HF config: +the per-key audit (loaded / transformed / ignored), the key remap onto the +runtime's parameter names and destination-owner routing. The tensors are placed by +``Glm5NextForCausalLM.load_weights``. +""" + +from __future__ import annotations + +import re +from collections import Counter +from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field +from typing import Any + +from transformers import PretrainedConfig + +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.models.checkpoints.hf.weight_mapper import HfWeightMapper +from tensorrt_llm._torch.models.modeling_utils import register_mapper +from tensorrt_llm._torch.pyexecutor.config_utils import unwrap_glm5_next_text_config +from tensorrt_llm.quantization.mode import QuantAlgo + +#: Checkpoint namespaces this bring-up deliberately does not load. These are +#: matched as exact dotted-component prefixes, never as substrings or globs: a +#: pattern like ``*visual*`` would also swallow a decoder weight that merely +#: contained the word, and the whole point of the audit is that nothing is +#: dropped by accident. +_VISION_PREFIX = "model.visual." +_LANGUAGE_PREFIX = "model.language_model." + +# --------------------------------------------------------------------------- +# Checkpoint key mapping +# --------------------------------------------------------------------------- + +#: The two hyper-connection sites are published as flat per-layer tensors +#: (``hc_attn_fn``) while the runtime holds each site as one ``mHC`` submodule +#: whose parameters are ``fn`` / ``base`` / ``scale``. Only the separator moves; +#: no tensor is reshaped, split, or fused. +_HC_RE = re.compile(r"^(model\.layers\.\d+\.)hc_(attn|ffn)_(fn|base|scale)$") + + +class Disposition: + """How a checkpoint tensor reaches (or does not reach) the runtime.""" + + #: Placed on a destination parameter unchanged. + LOADED = "loaded" + #: Placed after a shape/dtype/layout transformation (conv fusion, expert + #: stacking, block-FP8 dequantization, or a companion scale tensor). + TRANSFORMED = "transformed" + #: Deliberately not loaded, under an exact allowlisted namespace. + IGNORED = "ignored" + + +@dataclass +class Glm5NextWeightAudit: + """Exhaustive per-key accounting for a GLM-5.3-Flash checkpoint.""" + + #: destination module/parameter name -> source key + destinations: dict[str, str] = field(default_factory=dict) + #: source key -> disposition + disposition: dict[str, str] = field(default_factory=dict) + #: source key -> why it was transformed / ignored + reason: dict[str, str] = field(default_factory=dict) + #: keys that could not be placed at all -- always a hard error + unresolved: list[str] = field(default_factory=list) + + def counts(self) -> dict[str, int]: + return dict(Counter(self.disposition.values())) + + def keys_with(self, disposition: str) -> list[str]: + return sorted(k for k, d in self.disposition.items() if d == disposition) + + def reasons(self) -> dict[str, int]: + return dict(Counter(self.reason.values())) + + +def _ignored_reason(key: str, mtp_prefixes: Sequence[str]) -> str | None: + if key.startswith(_VISION_PREFIX): + return "vision tower weights are loaded by Glm5NextVLM, not the text decoder" + for prefix in mtp_prefixes: + if key.startswith(prefix): + return "MTP / next-n prediction layer is not enabled (no speculative_config)" + return None + + +def remap_glm5_next_key(key: str) -> str | None: + """Map one text-decoder checkpoint key to its runtime destination. + + Returns ``None`` for keys outside the text decoder (the caller decides + whether that is an allowlisted namespace or an error). + """ + if key == "lm_head.weight": + return key + if not key.startswith(_LANGUAGE_PREFIX): + return None + # model.language_model. -> model.: the runtime decoder is not + # nested inside a multimodal wrapper. + dest = "model." + key[len(_LANGUAGE_PREFIX) :] + + hc = _HC_RE.match(dest) + if hc is not None: + prefix, site, param = hc.groups() + return f"{prefix}hc_{site}.{param}" + return dest + + +def audit_glm5_next_checkpoint( + keys: Iterable[str], + config: PretrainedConfig, + *, + num_mtp_layers: int = 0, +) -> Glm5NextWeightAudit: + """Resolve every checkpoint key to exactly one destination and disposition. + + This is the Goal-1.2 contract in executable form. It is deliberately + analytic -- it needs only the safetensors index and the config, not 328 GB + of materialized weights -- so it can gate every later loading change + cheaply. + + ``num_mtp_layers`` is how many of the checkpoint's appended next-n + prediction (MTP) layers the model actually instantiates -- ``0`` for the + plain text model, ``1`` under one-model MTP speculative decoding. Those + layers' keys are placed on ``model.layers.{num_hidden_layers + i}.*`` + (the alias the speculative base class appends the draft layers under); + any remaining MTP layer stays an allowlisted ignore. + """ + text = unwrap_glm5_next_text_config(config) + num_layers = int(text.num_hidden_layers) + num_nextn = int(getattr(text, "num_nextn_predict_layers", 0) or 0) + if num_mtp_layers < 0 or num_mtp_layers > num_nextn: + raise ValueError( + f"glm5_next cannot load {num_mtp_layers} MTP layers; the checkpoint " + f"declares num_nextn_predict_layers={num_nextn}" + ) + # MTP layers are appended immediately after the decoder stack; the first + # ``num_mtp_layers`` of them are real destinations, the rest are ignored. + mtp_prefixes = tuple( + f"{_LANGUAGE_PREFIX}layers.{num_layers + i}." for i in range(num_mtp_layers, num_nextn) + ) + + audit = Glm5NextWeightAudit() + + for key in keys: + ignored = _ignored_reason(key, mtp_prefixes) + if ignored is not None: + audit.disposition[key] = Disposition.IGNORED + audit.reason[key] = ignored + continue + + dest = remap_glm5_next_key(key) + if dest is None: + audit.unresolved.append(key) + continue + + if dest.endswith(".weight_scale_inv"): + audit.disposition[key] = Disposition.TRANSFORMED + audit.reason[key] = "block-FP8 128x128 weight scale" + audit.destinations[dest] = key + continue + + if ".mlp.experts." in dest: + audit.disposition[key] = Disposition.TRANSFORMED + audit.reason[key] = "routed expert stacked into the fused MoE layout" + audit.destinations[dest] = key + continue + + audit.disposition[key] = Disposition.LOADED + audit.destinations[dest] = key + + return audit + + +# --------------------------------------------------------------------------- +# Checkpoint quantization +# --------------------------------------------------------------------------- + + +def glm5_next_is_quantized(model_config: ModelConfig[PretrainedConfig]) -> bool: + """Whether construction uses the checkpoint's block-FP8 form. + + The runtime constructs models through ``AutoModelForCausalLM.from_config``, + which calls ``cls(model_config)`` with no further arguments -- so the + quantization decision must live on the ``ModelConfig`` itself, exactly + where ``ModelConfig.from_pretrained`` puts it when it reads the + checkpoint's ``quantization_config`` (``weight_block_size=[128,128]`` maps + to ``FP8_BLOCK_SCALES``). A constructor flag that defaulted to bf16 would + make the runtime path build a model the loader must reject. + + This checkpoint is published in exactly one quantized form; any other + non-None algorithm on the config is a configuration error, not a request + for a different build. + """ + quant = getattr(model_config, "quant_config", None) + if quant is None or quant.quant_algo is None: + return False + if quant.quant_algo != QuantAlgo.FP8_BLOCK_SCALES: + raise ValueError( + "glm5_next supports only the published FP8_BLOCK_SCALES checkpoint " + f"form or unquantized bf16 modules; got quant_algo={quant.quant_algo}" + ) + return True + + +def _destination_owner(dest: str, num_layers: int) -> Any: + """Which materialization unit owns ``dest``: a layer index or a named part.""" + match = re.match(r"^model\.layers\.(\d+)\.", dest) + if match is not None: + index = int(match.group(1)) + return index if index < num_layers else None + if dest.startswith("model.embed_tokens."): + return "embed" + if dest.startswith("model.norm."): + return "norm" + if dest.startswith("lm_head."): + return "head" + return None + + +@register_mapper("HF", "Glm5NextForConditionalGeneration") +@register_mapper("HF", "Glm5NextForCausalLM") +class Glm5NextHfWeightMapper(HfWeightMapper): + """Checkpoint-key mapper for GLM-5.3-Flash (``glm5_next``). + + The HF checkpoint is a multimodal ``Glm5NextForConditionalGeneration`` + tree; the text model loads its ``model.language_model.*`` subtree with an + audited 1:1 placement (see :func:`audit_glm5_next_checkpoint`). This class + is the mapping half of that loader: which keys are ignored (vision tower, + surplus MTP layers), where each remaining key lands (:meth:`destination`), + and which materialization unit owns it (:meth:`owner`). The model's + ``load_weights`` owns the placement itself, so the generic module-name + callbacks of :class:`HfWeightMapper` are not used for this architecture. + """ + + def audit(self, keys: Iterable[str]) -> Glm5NextWeightAudit: + """Resolve every checkpoint key against the bound model's destinations.""" + num_mtp_layers = len(getattr(self.model, "mtp_layers", ())) + return audit_glm5_next_checkpoint( + keys, self.config.pretrained_config, num_mtp_layers=num_mtp_layers + ) + + @staticmethod + def destination(key: str) -> str | None: + """Runtime parameter name for a checkpoint key (``None``: not loaded).""" + return remap_glm5_next_key(key) + + @staticmethod + def owner(dest: str, num_layers: int) -> Any: + """Materialization owner of a destination: a layer index or a named part.""" + return _destination_owner(dest, num_layers) diff --git a/tensorrt_llm/_torch/models/checkpoints/hf/weight_loader.py b/tensorrt_llm/_torch/models/checkpoints/hf/weight_loader.py index 3eeae6ceb74a..b7cf6e65a648 100644 --- a/tensorrt_llm/_torch/models/checkpoints/hf/weight_loader.py +++ b/tensorrt_llm/_torch/models/checkpoints/hf/weight_loader.py @@ -58,7 +58,8 @@ _SUPPORTED_REQUESTED_IO_POLICIES = (_AUTO_IO_POLICY, ) + _SUPPORTED_IO_POLICIES # Model families whose checkpoints are too large to materialize in host RAM; # their models stream rank-local slices out of the lazy mmapped handles. -_LAZY_SAFETENSORS_MODEL_TYPES = ("kimi_k3", "kimi_linear") +_LAZY_SAFETENSORS_MODEL_TYPES = ("kimi_k3", "kimi_linear", "glm5_next", + "glm5_next_text") # Default to a single cached checkpoint: each entry pins a full copy of the # raw weights in CPU RAM, so callers wanting cross-model caching must opt in # via TRTLLM_HF_WEIGHT_CACHE_MAX_ENTRIES. diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next.py b/tensorrt_llm/_torch/models/modeling_glm5_next.py new file mode 100644 index 000000000000..ffb87dd717ac --- /dev/null +++ b/tensorrt_llm/_torch/models/modeling_glm5_next.py @@ -0,0 +1,2587 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""GLM-5.3-Flash text decoder and one-model MTP support. + +The decoder combines KDA recurrent attention, pool-compressed sparse MLA, +clamped-SwiGLU experts, and hyper-connections. The composite checkpoint's +``text_config`` defines the layer schedule; vision weights are not loaded. +""" + +from __future__ import annotations + +import os +import re +from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field +from typing import TYPE_CHECKING, Any, Callable + +import torch +from torch import nn +from transformers import PretrainedConfig + +from ...logger import logger +from ...mapping import Mapping +from ...models.modeling_utils import QuantConfig +from ..attention.backends.interface import ( + AttentionForwardArgs, + AttentionInputType, + AttentionMetadata, +) +from ..attention.backends.sparse.glm_kpool import ( + Glm5NextCacheManager, + Glm5NextMamba2Metadata, + GlmKpoolBackendForwardArgs, + GlmKpoolSparseParams, +) +from ..attention.backends.utils import create_attention +from ..model_config import ModelConfig +from ..modules.decoder_layer import DecoderLayer +from ..modules.embedding import Embedding +from ..modules.gated_mlp import GatedMLP +from ..modules.kimi_kda.kimi_kda_mixer import KimiKDALinearAttention +from ..modules.layer_norm import LayerNorm +from ..modules.linear import Linear, TensorParallelMode +from ..modules.rms_norm import RMSNorm +from ..pyexecutor.config_utils import unwrap_glm5_next_text_config +from .checkpoints.hf.glm5_next_weight_mapper import ( + Disposition, + Glm5NextHfWeightMapper, + Glm5NextWeightAudit, + glm5_next_is_quantized, +) +from .modeling_deepseekv3 import DeepseekV3Gate +from .modeling_speculative import SpecDecOneEngineForCausalLM +from .modeling_utils import DecoderModel, register_auto_model + +if TYPE_CHECKING: + from ..distributed import AllReduceStrategy + from ..modules.mamba.mamba2_metadata import Mamba2Metadata + from ..modules.mhc.hyper_connection import mHC + from ..pyexecutor.kv_cache.mamba_cache_manager import MambaHybridCacheManagerV2 + from .checkpoints.base_weight_mapper import BaseWeightMapper + + +# --------------------------------------------------------------------------- +# Literal schedule vocabulary +# --------------------------------------------------------------------------- + +LINEAR_ATTENTION = "linear_attention" +SPARSE_ATTENTION = "deepseek_sparse_attention" +DENSE_MLP = "dense" +SPARSE_MLP = "sparse" + + +def get_glm5_next_text_config(config: PretrainedConfig) -> PretrainedConfig: + """Return the text-decoder config, accepting either nesting level. + + The runtime resolves the model from the top-level ``Glm5NextConfig``, but + every decoder contract (schedules, ranks, MoE, HC) lives on + ``text_config``. Callers may already hold the inner config, so this is + idempotent. + """ + return unwrap_glm5_next_text_config(config) + + +@dataclass(frozen=True) +class Glm5NextSchedule: + """The two literal per-layer dispatch lists, validated against each other.""" + + attention: tuple[str, ...] + mlp: tuple[str, ...] + + @property + def num_layers(self) -> int: + return len(self.attention) + + def attention_indices(self, kind: str) -> tuple[int, ...]: + return tuple(i for i, t in enumerate(self.attention) if t == kind) + + def mlp_indices(self, kind: str) -> tuple[int, ...]: + return tuple[int, ...](i for i, t in enumerate(self.mlp) if t == kind) + + +def resolve_glm5_next_schedule(config: PretrainedConfig) -> Glm5NextSchedule: + """Read and cross-validate the literal dispatch lists. + + Raises on any disagreement between the three redundant encodings. A model + whose attention schedule is inferred from a cadence, or whose MLP schedule + is inferred from ``first_k_dense_replace``, would silently place the wrong + module (and therefore the wrong cache descriptor) at some layer; the config + states both lists explicitly, so there is no reason to guess. + """ + text = get_glm5_next_text_config(config) + num_layers = int(text.num_hidden_layers) + + attention = tuple(text.layer_types) + mlp = tuple(text.mlp_layer_types) + + for name, values, allowed in ( + ("layer_types", attention, {LINEAR_ATTENTION, SPARSE_ATTENTION}), + ("mlp_layer_types", mlp, {DENSE_MLP, SPARSE_MLP}), + ): + if len(values) != num_layers: + raise ValueError( + f"glm5_next {name} has {len(values)} entries but num_hidden_layers={num_layers}" + ) + unknown = sorted(set(values) - allowed) + if unknown: + raise ValueError(f"glm5_next {name} contains unsupported entries {unknown}") + + schedule = Glm5NextSchedule(attention=attention, mlp=mlp) + + # Third, redundant encoding of the attention schedule. It is not used for + # dispatch, but a disagreement means the checkpoint is not the variant this + # bring-up was validated against. + linear_attn_config = getattr(text, "linear_attn_config", None) or {} + kda_layers = linear_attn_config.get("kda_layers") + full_attn_layers = linear_attn_config.get("full_attn_layers") + if kda_layers is not None: + if tuple(kda_layers) != schedule.attention_indices(LINEAR_ATTENTION): + raise ValueError("glm5_next linear_attn_config.kda_layers disagrees with layer_types") + if full_attn_layers is not None: + if tuple(full_attn_layers) != schedule.attention_indices(SPARSE_ATTENTION): + raise ValueError( + "glm5_next linear_attn_config.full_attn_layers disagrees with layer_types" + ) + + # first_k_dense_replace is asserted against the literal list, not used to + # build it. + first_k_dense = getattr(text, "first_k_dense_replace", None) + if first_k_dense is not None: + expected_dense = tuple(range(int(first_k_dense))) + if schedule.mlp_indices(DENSE_MLP) != expected_dense: + raise ValueError( + f"glm5_next mlp_layer_types dense entries " + f"{schedule.mlp_indices(DENSE_MLP)} disagree with " + f"first_k_dense_replace={first_k_dense}" + ) + + return schedule + + +def glm5_next_allreduce_strategy() -> AllReduceStrategy: + """The ``AllReduceStrategy`` for every TP collective this model owns. + + Default ``ONESHOT``: the fused one-shot Lamport kernel (15-20 us for the + decode-sized [tokens, hidden] messages here, vs 20-75 us for NCCL's LL + ring at c=8..64; ``MIN_LATENCY`` switches to the two-shot kernel from a + few hundred tokens, which measured 38 us vs 19 us one-shot for the + 256-token speculative-verify messages). Messages above + :attr:`Glm5NextAllReduce.SMALL_MAX_TOKENS` go to NCCL regardless. The + bring-up pinned ``NCCL`` because the ``AUTO`` *autotuner* raced at TP4 + decode; a fixed strategy never enters the autotuner, so the race does not + apply. ``TLLM_GLM5_ALLREDUCE`` selects another strategy by name + (``NCCL``, ``MIN_LATENCY``, ``TWOSHOT``, ``AUTO``, ``NCCL_SYMMETRIC``) for + A/B measurement and as the escape hatch. + """ + from ..distributed import AllReduceStrategy + + name = os.environ.get("TLLM_GLM5_ALLREDUCE", "ONESHOT").strip().upper() + try: + return AllReduceStrategy[name] + except KeyError as exc: + raise ValueError( + f"TLLM_GLM5_ALLREDUCE={name!r} is not an AllReduceStrategy name: " + f"{[m.name for m in AllReduceStrategy]}" + ) from exc + + +def glm5_next_tp_reduces(mapping: Mapping | None) -> bool: + """Whether a TP branch output is a partial that needs one all-reduce. + + Under attention data parallelism every rank runs its own batch through + replicated attention / dense weights, so nothing is reduced there (the + fused MoE does its own dispatch/combine from ``all_rank_num_tokens``). + """ + return ( + mapping is not None + and int(getattr(mapping, "tp_size", 1) or 1) > 1 + and not bool(getattr(mapping, "enable_attention_dp", False)) + ) + + +def glm5_next_attention_mapping(mapping: Mapping | None) -> Mapping | None: + """The Mapping the attention projections shard over: the model's, or a + TP=1 view of it under attention DP (heads replicated per rank) -- the + same remap :class:`~tensorrt_llm._torch.attention.mla.MLA` applies.""" + if mapping is None or not getattr(mapping, "enable_attention_dp", False): + return mapping + return Mapping( + world_size=mapping.pp_size * mapping.tp_size, + tp_size=1, + pp_size=mapping.pp_size * mapping.tp_size, + rank=mapping.rank, + gpus_per_node=mapping.gpus_per_node, + enable_attention_dp=True, + ) + + +class Glm5NextAllReduce(nn.Module): + """TP all-reduce whose strategy follows the message size at call time. + + The fused one-shot Lamport kernel (``ONESHOT``) is 15-20 us for + decode-sized ``[tokens, hidden]`` messages where NCCL's LL ring takes + 20-75 us, but from ~1K tokens up the fused kernels are slower than NCCL + (56 vs 46 us at 1K tokens; 408 vs 355 us two-shot vs NCCL at 8K). One + ``AllReduce`` per strategy, chosen by the token count -- a Python int, so a + captured decode graph has a fixed choice. Parameter-free. + """ + + #: Messages with more tokens than this go to NCCL. Measured on this node + #: (TP4, hidden 4096, bf16): the fused kernels win up to ~256 tokens + #: (15-19 us vs 20-27 us NCCL), tie around 512, and lose from 1024 tokens + #: (56 vs 46 us) upward. + SMALL_MAX_TOKENS = 512 + + def __init__(self, mapping: Mapping, dtype: torch.dtype = torch.bfloat16) -> None: + super().__init__() + from ..distributed import AllReduce, AllReduceStrategy + + strategy = glm5_next_allreduce_strategy() + self.small = AllReduce(mapping=mapping, strategy=strategy, dtype=dtype) + self.large = ( + self.small + if strategy == AllReduceStrategy.NCCL + else AllReduce(mapping=mapping, strategy=AllReduceStrategy.NCCL, dtype=dtype) + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + x = x.contiguous() + return (self.small if x.shape[0] <= self.SMALL_MAX_TOKENS else self.large)(x) + + +# --------------------------------------------------------------------------- +# Model discovery +# --------------------------------------------------------------------------- + + +def _normalize_glm5_next_top_config(config: PretrainedConfig) -> None: + """Give the composite multimodal config the fields the runtime reads. + + The checkpoint's top-level ``Glm5NextConfig`` carries no + ``num_hidden_layers`` or ``torch_dtype`` -- both live on ``text_config`` -- + but ``DecoderModel``/``DecoderModelForCausalLM`` and executor capacity + planning read them from the config they are handed. Copy them up once, + from the text config, rather than teaching every runtime consumer about + the wrapper. Values already present are left alone. + """ + text = get_glm5_next_text_config(config) + if getattr(config, "num_hidden_layers", None) is None: + config.num_hidden_layers = int(text.num_hidden_layers) + if getattr(config, "torch_dtype", None) is None: + config.torch_dtype = getattr(text, "torch_dtype", None) or torch.bfloat16 + # The one-model MTP drafter (``MTPForCausalLM``) reads the checkpoint's + # next-n layer count from the config it is handed, which is this top-level + # composite one; the field lives on ``text_config``. + if getattr(config, "num_nextn_predict_layers", None) is None: + config.num_nextn_predict_layers = int(getattr(text, "num_nextn_predict_layers", 0) or 0) + + +@dataclass(frozen=True) +class Glm5NextContextRows: + """Device-side row schedule of the packed context tokens of one forward. + + Built once per forward from the host ``cu_seqlens``/``cached_lens`` and + shared by every sparse layer, so the prefill path issues one kernel per + stage for all context requests instead of a Python loop per request. + ``positions[i]`` is packed token ``i``'s cache position, ``request_ids[i]`` + its executor request index (the batch's block-table row); + ``final_positions``/``final_request_ids`` list the pool-final positions + this forward completes (one per pool, so pool-key refreshes never race). + """ + + positions: torch.Tensor + request_ids: torch.Tensor + final_positions: torch.Tensor + final_request_ids: torch.Tensor + + @classmethod + def build( + cls, + cu_seqlens: Sequence[int], + cached_lens: Sequence[int], + kpool: int, + device: torch.device, + ) -> Glm5NextContextRows: + positions: list[int] = [] + request_ids: list[int] = [] + finals: list[int] = [] + final_ids: list[int] = [] + for i in range(len(cu_seqlens) - 1): + length = int(cu_seqlens[i + 1]) - int(cu_seqlens[i]) + cached = int(cached_lens[i]) + positions.extend(range(cached, cached + length)) + request_ids.extend([i] * length) + first_final = cached + (-(cached + 1)) % kpool + for pos in range(first_final, cached + length, kpool): + finals.append(pos) + final_ids.append(i) + as_dev = lambda vals, dtype: torch.tensor(vals, dtype=dtype, device=device) # noqa: E731 + return cls( + positions=as_dev(positions, torch.long), + request_ids=as_dev(request_ids, torch.int32), + final_positions=as_dev(finals, torch.long), + final_request_ids=as_dev(final_ids, torch.int32), + ) + + +@dataclass +class Glm5NextRuntimeContext: + """Per-forward schedule arguments derived once from AttentionMetadata. + + The sparse layers take *schedule* values (request boundaries, positions) + plus the prepared ``metadata`` itself -- their attention backend derives + every cache pool, block table, and visible length from that metadata. The + KDA layers consume the prepared metadata directly (the shared Kimi KDA + mixer reads its pools and slot ids from it). This object is built + once per model forward; requests are packed context-first, matching the + executor's batch layout. + """ + + manager: Any + num_contexts: int + num_ctx_tokens: int + num_generations: int + ctx_cu_seqlens: list[int] + cached_lens: list[int] + state_indices: torch.Tensor + #: Per-request visible lengths (cached + this step's tokens) as a device + #: tensor. The decode path consumes ONLY device values so that captured + #: CUDA graphs replay against prepare()-refreshed buffers rather than + #: Python ints baked in at capture time. + kv_lens: torch.Tensor + #: The engine's prepared, typed attention metadata this context was + #: derived from; the sparse layers' attention backend consumes it as the + #: single source of cache state. + metadata: AttentionMetadata + #: Tokens per generation request in this step: ``1`` for plain decode, + #: ``1 + runtime_draft_len`` while a speculative-decoding target verifies + #: its draft tokens (the executor packs every generation request to the + #: same width, so this is one host int, fixed at CUDA-graph capture). + gen_tokens_per_request: int = 1 + #: Context-row schedules by pool size, built on first use and shared by + #: all sparse layers of this forward (prefill only, never captured). + ctx_rows_cache: dict[int, Glm5NextContextRows] = field(default_factory=dict) + + def context_rows(self, kpool: int, device: torch.device) -> Glm5NextContextRows: + rows = self.ctx_rows_cache.get(kpool) + if rows is None: + rows = Glm5NextContextRows.build( + self.ctx_cu_seqlens, self.cached_lens[: self.num_contexts], kpool, device + ) + self.ctx_rows_cache[kpool] = rows + return rows + + @property + def gen_phase(self) -> str: + """Which attention entry point the generation rows take. + + ``"decode"`` is the single-token path; ``"verify"`` is the + multi-token speculative verification path (the fused Kimi KDA replay + kernel: the state is committed after the golden token and the drafts + are cached for replay once the sampler has decided the accepted count). + """ + return "decode" if self.gen_tokens_per_request == 1 else "verify" + + def mixed_kwargs(self, layer_idx: int) -> dict[str, Any]: + """Arguments of the sparse layers' ``forward_mixed`` for a + context+generation batch. + + The attention module splits the packed tokens at ``num_ctx_tokens`` + and runs its context rows through the prefill kernels and its + generation rows through the decode kernels; everything outside + attention runs once over the whole batch (the vLLM layout). The KDA + layers do this split inside the shared mixer from the metadata. + """ + return { + "num_ctx_tokens": self.num_ctx_tokens, + "prefill": self.sparse_kwargs(layer_idx, "prefill"), + "decode": self.sparse_kwargs(layer_idx, "decode"), + } + + def sparse_kwargs(self, layer_idx: int, phase: str) -> dict[str, Any]: + # Schedule only: the backend (keyed by its own layer_idx) derives the + # slot-indexed latent/indexer views, block tables, and lengths from + # the prepared metadata itself. + del layer_idx + kwargs: dict[str, Any] = {"metadata": self.metadata} + if phase == "prefill": + kwargs.update( + cached_lens=self.cached_lens[: self.num_contexts], + cu_seqlens=self.ctx_cu_seqlens, + ctx_rows_fn=self.context_rows, + ) + else: + kwargs.update(kv_lens=self.kv_lens[self.num_contexts :]) + if phase == "verify": + kwargs.update(tokens_per_request=self.gen_tokens_per_request) + return kwargs + + +def _glm5_gen_tokens_per_request(attn_metadata: AttentionMetadata, num_generations: int) -> int: + """Tokens per generation request, from the metadata's host token counts. + + Plain decode has one; a speculative-decoding target verifying drafts has + ``1 + runtime_draft_len``, uniformly across the generation rows (the + executor pads every drafted request to the same width). Derived from + host ints only, so it is a fixed shape parameter inside CUDA graphs. + """ + if num_generations <= 0: + return 1 + num_tokens = getattr(attn_metadata, "num_tokens", None) + if num_tokens is None: + # Harness carriers without the runtime's cached token count: the + # host seq_lens carry the same information (never a captured path). + num_tokens = int(attn_metadata.seq_lens.sum()) + gen_tokens = int(num_tokens) - int(attn_metadata.num_ctx_tokens) + if gen_tokens <= 0 or gen_tokens % num_generations: + raise ValueError( + f"glm5_next: {gen_tokens} generation tokens do not split evenly over " + f"{num_generations} generation requests" + ) + return gen_tokens // num_generations + + +def glm5_next_visible_lens(attn_metadata: AttentionMetadata, batch: int) -> torch.Tensor | None: + """Per-request visible lengths (``cached + this step's tokens``) as int64. + + Prefers the attention metadata's own ``kv_lens_cuda`` over the + ``Glm5NextMamba2Metadata`` copy. Both hold the same values after + ``prepare()``, but only ``kv_lens_cuda`` receives the engine's in-graph + corrections: under the overlap scheduler with speculative decoding the + host prepares generation requests as if every draft of the previous step + had been accepted, and ``_preprocess_inputs`` subtracts the rejected + count on device (``previous_kv_lens_offsets_cuda``) right before the + forward; the speculative worker likewise rewinds it between draft steps. + Reading the host-derived copy there positions the new latent/indexer rows + past the real prefix and attends stale page contents -- observed as + non-deterministic MTP output under config E. Returns ``None`` when the + metadata carries no ``kv_lens_cuda`` (harness carriers), so callers fall + back to the GLM buffer. The int32 -> int64 cast is a device op with no + host sync, so it is legal inside CUDA-graph capture. + """ + live = getattr(attn_metadata, "kv_lens_cuda", None) + if live is None: + return None + return live[:batch].to(torch.long) + + +def build_glm5_next_runtime_context( + attn_metadata: AttentionMetadata, + *, + kv_lens_source: str = "glm", +) -> Glm5NextRuntimeContext: + """Derive the per-forward cache arguments from prepared metadata. + + Requires ``attn_metadata.prepare()`` to have run: that is what attaches + ``mamba_metadata`` (the manager is a ``BaseMambaCacheManager``) and fills + its batch-ordered ``state_indices``. ``cached_lens`` follows the runtime's + own convention -- tokens already in the cache, excluding the ones in this + step -- which is exactly what ``forward_prefill``/``forward_decode`` seed + and position from. + + Visible lengths come from :func:`glm5_next_visible_lens` (the metadata's + device-corrected ``kv_lens_cuda``) whenever the metadata carries it, for + both the target and the MTP draft layer; the ``Glm5NextMamba2Metadata`` + copy is the fallback for harness carriers. ``kv_lens_source`` is kept for + call-site documentation (``"metadata"`` marks the draft layer, whose + lengths the speculative worker rewinds in place between draft steps) and + to reject unknown values; it no longer changes the source when + ``kv_lens_cuda`` is present. + """ + manager = attn_metadata.kv_cache_manager + if manager is None: + raise ValueError("glm5_next requires a kv cache manager; got None") + mamba_metadata = attn_metadata.mamba_metadata + if mamba_metadata is None or mamba_metadata is False: + raise ValueError( + "glm5_next requires mamba_metadata; call attn_metadata.prepare() " + "with the Glm5NextCacheManager attached" + ) + if kv_lens_source not in ("glm", "metadata"): + raise ValueError(f"glm5_next: unknown kv_lens_source {kv_lens_source!r}") + batch = int(attn_metadata.seq_lens.shape[0]) + num_contexts = int(attn_metadata.num_contexts) + num_generations = batch - num_contexts + gen_tokens_per_request = _glm5_gen_tokens_per_request(attn_metadata, num_generations) + + if getattr(mamba_metadata, "glm_block_tables", None) is not None: + # Persistent path: every tensor below is a prepare()-refreshed buffer + # slice, so this function does no allocation, no H2D, and no host + # sync -- it is safe to run inside CUDA graph capture, and replays + # read the refreshed values at the same addresses. + kv_lens = glm5_next_visible_lens(attn_metadata, batch) + if kv_lens is None: + if kv_lens_source == "metadata": + raise ValueError( + "glm5_next draft layer needs attn_metadata.kv_lens_cuda (the " + "TRTLLM metadata family); the attached metadata has none" + ) + kv_lens = mamba_metadata.glm_kv_lens[:batch] + return Glm5NextRuntimeContext( + manager=manager, + num_contexts=num_contexts, + num_ctx_tokens=int(attn_metadata.num_ctx_tokens), + num_generations=num_generations, + ctx_cu_seqlens=mamba_metadata.glm_ctx_cu_seqlens, + cached_lens=mamba_metadata.glm_cached_lens_host, + state_indices=mamba_metadata.state_indices[:batch], + kv_lens=kv_lens, + metadata=attn_metadata, + gen_tokens_per_request=gen_tokens_per_request, + ) + + # Legacy eager construction, kept for harnesses whose fake managers do + # not attach the GLM metadata buffers. It allocates and copies, so it + # must never run inside a captured region. (The sparse backend applies + # the same rule to its own metadata-derived block tables.) + if getattr(attn_metadata, "is_cuda_graph", False): + raise RuntimeError( + "glm5_next CUDA-graph execution requires the Glm5NextCacheManager's " + "Glm5NextMamba2Metadata (persistent prepare()-refreshed buffers); " + "the attached mamba_metadata has no glm_block_tables" + ) + lens = attn_metadata.seq_lens.tolist() + kv_params = attn_metadata.kv_cache_params + if kv_params is None or kv_params.num_cached_tokens_per_seq is None: + raise ValueError("glm5_next requires kv_cache_params.num_cached_tokens_per_seq") + cached_lens = [int(n) for n in kv_params.num_cached_tokens_per_seq[:batch]] + + ctx_cu = [0] + for length in lens[:num_contexts]: + ctx_cu.append(ctx_cu[-1] + int(length)) + + device = torch.device("cuda", torch.cuda.current_device()) + kv_lens = torch.as_tensor( + [c + n for c, n in zip(cached_lens, lens)], dtype=torch.long, device=device + ) + + live = glm5_next_visible_lens(attn_metadata, batch) + if live is not None: + kv_lens = live + + return Glm5NextRuntimeContext( + manager=manager, + num_contexts=num_contexts, + num_ctx_tokens=int(attn_metadata.num_ctx_tokens), + num_generations=num_generations, + ctx_cu_seqlens=ctx_cu, + cached_lens=cached_lens, + state_indices=mamba_metadata.state_indices[:batch], + kv_lens=kv_lens, + metadata=attn_metadata, + gen_tokens_per_request=gen_tokens_per_request, + ) + + +@register_auto_model("Glm5NextForCausalLM") +class Glm5NextForCausalLM(SpecDecOneEngineForCausalLM): + """GLM-5.3-Flash text decoder with an optional one-model MTP drafter. + + The speculative base class owns logits processing and the draft/verify + lifecycle. This class narrows the composite config and loads each rank's + checkpoint shard, including the optional appended MTP layer. Vision weights + are excluded explicitly by :func:`audit_glm5_next_checkpoint`. + """ + + @property + def mamba_metadata_cls(self) -> type[Mamba2Metadata]: + """Metadata with paged tables refreshed before CUDA graph replay.""" + return glm5_next_mamba_metadata_cls() + + def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: + text_config = get_glm5_next_text_config(model_config.pretrained_config) + if model_config.mapping.enable_attention_dp and model_config.spec_config is not None: + raise ValueError( + "glm5_next does not support attention DP with speculative decoding; " + "set enable_attention_dp=False when enabling MTP." + ) + # tie_word_embeddings is false on this checkpoint, so lm_head is a real + # weight rather than a view of the embedding; it is asserted, not assumed. + if bool(getattr(text_config, "tie_word_embeddings", False)): + raise ValueError( + "glm5_next was validated with untied output embeddings; a tied " + "checkpoint would need lm_head to alias embed_tokens" + ) + _normalize_glm5_next_top_config(model_config.pretrained_config) + super().__init__( + Glm5NextModel(model_config), + model_config, + hidden_size=int(text_config.hidden_size), + vocab_size=int(text_config.vocab_size), + ) + # ``config`` is a base-class property (the top-level composite config); + # the narrowed decoder contract lives here. + self.text_config = text_config + self.schedule = resolve_glm5_next_schedule(model_config.pretrained_config) + # One-model MTP: the base class built ``draft_model.mtp_layers`` (one + # Glm5NextMTP at layer_idx 45) and the speculative worker. Alias the + # draft layer(s) onto ``model.layers[45:]`` so the checkpoint's + # ``model.layers.45.*`` keys are placed by the same exact loader, the + # same projection swap, and the same per-owner materialization. + self.mtp_layers: tuple[nn.Module, ...] = () + spec_config = getattr(model_config, "spec_config", None) + if spec_config is not None and spec_config.spec_dec_mode.is_mtp_one_model(): + mtp_layers = tuple(self.draft_model.mtp_layers) + if len(mtp_layers) != 1: + raise ValueError( + f"glm5_next builds exactly one MTP layer (num_nextn_predict_layers=1); " + f"the drafter constructed {len(mtp_layers)}" + ) + self.model.layers.extend(mtp_layers) + self.mtp_layers = mtp_layers + # Derived from the ModelConfig, never a constructor flag: the runtime's + # AutoModelForCausalLM.from_config calls cls(model_config) and nothing + # else, so this is the only place the decision can live. + self.quantized = glm5_next_is_quantized(model_config) + # One provenance line per rank: engine-scale runs (LLM API / serving) + # spawn MPI workers whose model objects the driver cannot introspect, + # so the resolved production stack is published through the worker log. + attn_backends = sorted( + { + type(layer.self_attn.attn_backend).__name__ + for layer in self.model.layers + if isinstance(layer.self_attn, Glm5NextSparseAttention) + } + ) + moe_backends = sorted( + { + layer.mlp.moe_backend_name + for layer in self.model.layers + if isinstance(layer.mlp, Glm5NextMoE) + } + ) + logger.info( + f"glm5_next runtime stack: sparse_attention={attn_backends}, " + f"moe_backend={moe_backends}, quantized={self.quantized}, " + f"kv_cache_manager=V2 (Glm5NextCacheManager), " + f"mtp_layers={len(self.mtp_layers)}" + ) + + @property + def num_hidden_layers(self) -> int: + return self.schedule.num_layers + + def apply_quant_config_exclude_modules(self) -> None: + """Base-class exclusion by runtime name, extended to the fused Linears. + + A fused projection (``FUSED_LINEARS`` on its owner) is spelled in the + checkpoint -- and in ``modules_to_not_convert`` -- as its separate + source tensors, so the verdict is taken from the sources: all excluded + keeps the fused module BF16, none excluded keeps it quantized, a mix + is not representable and is rejected. + """ + super().apply_quant_config_exclude_modules() + quant_config = self.model_config.quant_config + if quant_config is None or quant_config.exclude_modules is None: + return + new_config = QuantConfig(kv_cache_quant_algo=quant_config.kv_cache_quant_algo) + for name, module in self.named_modules(): + for fused_attr, sources in getattr(type(module), "FUSED_LINEARS", ()): + fused = module.get_submodule(fused_attr) + if getattr(fused, "quant_config", None) is None: + continue + prefix = f"{name}." if name else "" + verdicts = [ + quant_config.is_module_excluded_from_quantization(prefix + src) + for src in sources + ] + if all(verdicts): + fused.quant_config = new_config + fused._weights_created = False + elif any(verdicts): + raise ValueError( + f"glm5_next {prefix}{fused_attr}: fused sources disagree on " + f"quantization ({dict(zip(sources, verdicts))}); fusing tensors " + "with different quantization is not representable" + ) + + def infer_max_seq_len(self) -> int: + """Max sequence length the runtime sizes KV/mamba caches for. + + The executor calls this during capacity planning. GLM-5.3-Flash declares + ``max_position_embeddings=1048576`` and is fully NoPE (no rope-factor + scaling), so the value is the text config's directly. + """ + return int(self.text_config.max_position_embeddings) + + @classmethod + def get_preferred_kv_cache_manager_version(cls, pretrained_config=None) -> str: + """Opt this model into ``KVCacheManagerV2``. + + The hybrid latent-KV + pool-indexer + recurrent/conv state is owned by a + single ``Glm5NextCacheManager`` (a ``MambaHybridCacheManagerV2`` subclass, + :func:`glm5_next_cache_manager_cls`); V1 cannot express it. This is the + ``"auto"`` -> V2 resolution hook the runtime consults. + """ + return "V2" + + def attention_type(self, layer_idx: int) -> str: + """The literal attention module type for ``layer_idx``.""" + return self.schedule.attention[layer_idx] + + def mlp_type(self, layer_idx: int) -> str: + """The literal feed-forward module type for ``layer_idx``.""" + return self.schedule.mlp[layer_idx] + + def audit_checkpoint(self, keys: Iterable[str]) -> Glm5NextWeightAudit: + """Resolve every checkpoint key against this model's destinations.""" + mapper = Glm5NextHfWeightMapper() + mapper.init_model_and_config(self, self.model_config) + return mapper.audit(keys) + + # -- whole-model materialization -------------------------------------- + + def load_weights( + self, + weights: Any, + weight_mapper: BaseWeightMapper | None = None, + *, + device_map: dict[Any, Any] | None = None, + ) -> None: + """Materialize and fill the whole text model from a raw checkpoint. + + ``weight_mapper`` is the :class:`Glm5NextHfWeightMapper` registered for + this architecture (the runtime's ``ModelLoader`` hands over the + initialized instance; harnesses may omit it and one is created here). + It owns every checkpoint-key decision -- the audit, the key remap and + the destination owner -- while this method owns the materialization: + exact-shape placement one owner at a time, then the decode fusions. + A generic HF mapper is rejected: its module-name rules cannot place + this checkpoint. + + ``weights`` is any mapping from checkpoint key to the tensor **as + stored** -- e4m3 payloads and their FP32 block scales are copied + verbatim, never dequantized, because excluded modules are published in + BF16 and quantized ones in e4m3 with a scale, with no overlap. That + makes the load a 1:1 placement and keeps the resident model at the + checkpoint's own 328 GB. + + ``device_map`` maps each owner -- a layer index, or ``"embed"``, + ``"norm"``, ``"head"`` -- to a device. The model is expected to have + been constructed on ``meta``: each owner is materialized directly onto + its target device and filled immediately, so peak memory is one layer + above the final footprint rather than a second full copy. + + A checkpoint tensor that finds no parameter, and a parameter that + receives no tensor, are both errors: either one leaves a model that + still runs and still looks plausible. + """ + if weight_mapper is None: + weight_mapper = Glm5NextHfWeightMapper() + weight_mapper.init_model_and_config(self, self.model_config) + elif not isinstance(weight_mapper, Glm5NextHfWeightMapper): + raise ValueError( + "glm5_next needs its registered Glm5NextHfWeightMapper; got " + f"{type(weight_mapper).__name__}" + ) + if not self.quantized: + raise ValueError( + "glm5_next whole-model loading requires the block-FP8 build " + "(quantized=True). Dequantizing all 288 experts of 42 routed " + "layers to bf16 would double the resident model to ~656 GB and " + "move it four times further from the source's own arithmetic." + ) + # Decoder stack plus the aliased MTP draft layer(s), if any: owners + # ``45..`` are the draft layers, placed from the checkpoint's own + # ``layers.45.*`` keys instead of being allowlisted away. + num_layers = self.schedule.num_layers + len(self.mtp_layers) + audit = weight_mapper.audit(list(weights.keys())) + + # Each owner is materialized and filled on its own, so peak memory is + # one layer above the final footprint rather than a second full copy. + targets: dict[Any, tuple[nn.Module, str]] = { + **{i: (self.model.layers[i], f"model.layers.{i}.") for i in range(num_layers)}, + "embed": (self.model.embed_tokens, "model.embed_tokens."), + "norm": (self.model.norm, "model.norm."), + "head": (self.lm_head, "lm_head."), + } + # Owners pruned by pipeline parallelism (`__pp_init__` cleared their + # parameters): their checkpoint keys belong to another rank. + remote = { + owner + for owner, (module, _) in targets.items() + if getattr(module, "_weights_removed", False) + } + + by_owner: dict[Any, list[tuple[str, str]]] = {} + for key, disposition in audit.disposition.items(): + if disposition == Disposition.IGNORED: + continue + dest = weight_mapper.destination(key) + owner = weight_mapper.owner(dest, num_layers) if dest else None + if owner is None: + raise ValueError(f"glm5_next has no destination owner for {key!r} -> {dest!r}") + if owner in remote: + continue + by_owner.setdefault(owner, []).append((key, dest)) + if audit.unresolved: + raise ValueError(f"glm5_next cannot place {sorted(audit.unresolved)[:5]}") + + default_device = torch.device("cuda", torch.cuda.current_device()) + for owner, (module, prefix) in targets.items(): + if owner in remote: + continue + device = torch.device((device_map or {}).get(owner, default_device)) + module.to_empty(device=device) + self._fill_module(module, owner, prefix, by_owner.get(owner, []), weights, device) + # The shared KDA mixer's post-load kernel constants (built from the + # just-loaded local shards; the same step the Kimi K3 loader runs). + for layer in self.model.layers: + attn = getattr(layer, "self_attn", None) + if isinstance(attn, Glm5NextLinearAttention) and not getattr( + layer, "_weights_removed", False + ): + attn.finalize_weights() + logger.info(f"glm5_next loaded {len(targets) - len(remote)} owners") + + def _fill_module( + self, + module: nn.Module, + owner: Any, + prefix: str, + entries: Sequence[tuple[str, str]], + weights: Any, + device: torch.device, + ) -> None: + """Place one materialized owner's tensors. + + * A TensorRT-LLM ``Linear`` destination receives its checkpoint tensors + through the module's own ``load_weights`` (which owns the TP slicing + and the block-scale layout). Projections the runtime keeps fused + (``GatedMLP.gate_up_proj``, the sparse layers' ``[q_a | kv_a]`` and + the indexer's ``[wk | gate | weights_proj]``) collect their separate + checkpoint tensors first (:attr:`FUSED_LINEARS` on the owning module). + * Production routed experts go to the fused MoE layer's loader, + filtered to ``initial_local_expert_ids``. + * The KDA layer's tensors (the shared mixer's local-width modules) are + sliced to this rank's head range by the module itself. + * Everything else (norms, HC, router, embeddings) is an exact-shape + replicated copy. + """ + params = dict(module.named_parameters()) + params.update(dict(module.named_buffers())) + named_modules = dict(module.named_modules()) + kda = named_modules.get("self_attn") + if not isinstance(kda, Glm5NextLinearAttention): + kda = None + filled: set[str] = set() + + # source module path -> (fused Linear path, position in the concatenation) + fused_of: dict[str, tuple[str, int]] = {} + for mod_name, sub in named_modules.items(): + base = f"{mod_name}." if mod_name else "" + for fused_attr, sources in getattr(type(sub), "FUSED_LINEARS", ()): + for pos, src in enumerate(sources): + fused_of[base + src] = (base + fused_attr, pos) + if isinstance(sub, GatedMLP): + fused_of[base + "gate_proj"] = (base + "gate_up_proj", 0) + fused_of[base + "up_proj"] = (base + "gate_up_proj", 1) + linear_groups: dict[str, dict[str, Any]] = {} + fused_groups: dict[str, dict[int, dict[str, Any]]] = {} + + experts = getattr(getattr(module, "mlp", None), "experts", None) + local_expert_ids = set(experts.initial_local_expert_ids) if experts is not None else set() + moe_weights: dict[str, torch.Tensor] = {} + proj_to_w = {"gate_proj": "w1", "up_proj": "w3", "down_proj": "w2"} + + def materialize(t: Any) -> torch.Tensor: + # Lazy safetensors slice: indexing materializes only this tensor. + return t if torch.is_tensor(t) else t[:] + + for key, dest in sorted(entries): + local = dest.removeprefix(prefix) + lazy = weights[key] + + moe = _FUSED_MOE_RE.match(dest.removesuffix("_scale_inv").removesuffix(".weight")) + if moe is not None: + expert_id = int(moe.group("expert")) + if expert_id in local_expert_ids: # other EP ranks' bytes are never read + suffix = ".weight_scale_inv" if dest.endswith("_scale_inv") else ".weight" + moe_weights[f"{expert_id}.{proj_to_w[moe.group('proj')]}{suffix}"] = ( + materialize(lazy) + ) + continue + + if local.endswith(".weight_scale_inv"): + mod_path, label = local[: -len(".weight_scale_inv")], "weight_scale_inv" + else: + mod_path, _, label = local.rpartition(".") + if mod_path in fused_of: + fused_path, pos = fused_of[mod_path] + fused_groups.setdefault(fused_path, {}).setdefault(pos, {})[label] = materialize( + lazy + ) + continue + if local in fused_of: # a bare parameter (no ``.weight``) feeding a fused Linear + fused_path, pos = fused_of[local] + fused_groups.setdefault(fused_path, {}).setdefault(pos, {})["weight"] = materialize( + lazy + ) + continue + if isinstance(named_modules.get(mod_path), Linear): + linear_groups.setdefault(mod_path, {})[label] = materialize(lazy) + continue + + tensor = materialize(lazy) + if kda is not None and local.startswith("self_attn."): + tensor = kda.shard_checkpoint_tensor(local[len("self_attn.") :], tensor) + target = params.get(local) + if target is None: + raise KeyError(f"glm5_next has no parameter for {key!r} (destination {dest!r})") + if tuple(target.shape) != tuple(tensor.shape): + raise ValueError( + f"glm5_next {dest!r}: checkpoint shape {tuple(tensor.shape)} does not " + f"match parameter shape {tuple(target.shape)}" + ) + with torch.no_grad(): + target.copy_(tensor.to(device=device, dtype=target.dtype)) + filled.add(local) + + def mark_linear(mod_path: str) -> None: + for param_name in ("weight", "weight_scale", "bias"): + # mod_path is "" when the owner *is* the Linear (the LMHead). + name = f"{mod_path}.{param_name}" if mod_path else param_name + if name in params: + filled.add(name) + + for mod_path, group in linear_groups.items(): + named_modules[mod_path].load_weights([group]) + mark_linear(mod_path) + for fused_path, parts in fused_groups.items(): + dest_mod = named_modules[fused_path] + groups = [parts[i] for i in sorted(parts)] + if ( + isinstance(dest_mod, Linear) + and getattr(dest_mod.weights_loading_config, "weight_mode", None) is not None + and dest_mod.weights_loading_config.weight_mode.name == ("FUSED_GATE_UP_LINEAR") + ): + # GatedMLP: the Linear shards each half over this rank's + # intermediate range and stacks them [gate; up] itself. + dest_mod.load_weights(groups) + else: + # Replicated fusions: one row-concatenated tensor (and block + # scales), the order being the module's declared source order. + merged = {"weight": torch.cat([g["weight"] for g in groups], dim=0)} + if all("weight_scale_inv" in g for g in groups): + merged["weight_scale_inv"] = torch.cat( + [g["weight_scale_inv"] for g in groups], dim=0 + ) + dest_mod.load_weights([merged]) + mark_linear(fused_path) + + if experts is not None: + expected = 6 * len(local_expert_ids) + if len(moe_weights) != expected: + raise ValueError( + f"glm5_next owner {owner!r}: fused MoE collected {len(moe_weights)} expert " + f"tensors, expected {expected} (local experts {len(local_expert_ids)})" + ) + experts.load_weights([moe_weights]) + if hasattr(experts, "post_load_weights"): + experts.post_load_weights() + filled.update(n for n in params if n.startswith("mlp.experts.")) + + # FP8_BLOCK_SCALES' static activation scales: this checkpoint is + # activation_scheme='dynamic', so they are never read; zero them rather + # than leave to_empty garbage behind. + unused = {n for n in params if n.endswith(("input_scale", "inv_input_scale"))} + with torch.no_grad(): + for name in unused: + params[name].zero_() + unfilled = sorted(set(params) - filled - unused) + if unfilled: + raise ValueError(f"glm5_next owner {owner!r}: no checkpoint tensor reached {unfilled}") + + +# --------------------------------------------------------------------------- +# KDA linear attention +# --------------------------------------------------------------------------- + + +class Glm5NextLinearAttention(KimiKDALinearAttention): + """GLM-5.3-Flash KDA layer: the shared Kimi KDA mixer, configured. + + The recurrence, convolution, projections, mixed context+generation + batches and the speculative-verify replay kernel are all the shared + module's (``trtllm::kda_prefill`` / ``kda_decode`` / ``kda_mtp_decode`` + with the FLA fallbacks), reading the ``MambaHybridCacheManagerV2`` pools + the way Kimi K3 does. What GLM-5.3-Flash configures differently: + + * the **low-rank output gate** ``g_b_proj(g_a_proj(x))`` (the checkpoint + has no full-rank ``g_proj``), selected through the mixer's own + ``use_full_rank_gate`` config key and served by its fused + ``[f_a | g_a | b]`` / ``[f_b; g_b]`` decode projections; + * ``A_log`` / ``dt_bias`` are **published in fp32** and kept so. + + The exact-placement loader shards this rank's head range itself + (:meth:`shard_checkpoint_tensor`), as the projections are the mixer's own + local-width ``nn.Linear`` modules rather than Mapping-aware ``Linear``. + """ + + #: Checkpoint tensors sharded by this rank's head *channel* range on dim 0. + _CHANNEL_ROW_SHARDED = frozenset( + ( + "q_proj.weight", + "k_proj.weight", + "v_proj.weight", + "f_b_proj.weight", + "g_b_proj.weight", + "q_conv1d.weight", + "k_conv1d.weight", + "v_conv1d.weight", + "dt_bias", + ) + ) + #: Checkpoint tensors sharded by this rank's head range on dim 0. + _HEAD_ROW_SHARDED = frozenset(("b_proj.weight", "A_log")) + #: Replicated on every rank. + _REPLICATED = frozenset(("f_a_proj.weight", "g_a_proj.weight", "o_norm.weight")) + + def __init__( + self, + config: PretrainedConfig, + layer_idx: int, + dtype: torch.dtype = torch.bfloat16, + mapping: Mapping | None = None, + ) -> None: + del dtype # the shared mixer is bf16 (fp32 gate parameters and pools) + # The checkpoint's ``linear_attn_config`` does not name the gate rank: + # GLM-5.3-Flash's output gate is always the low-rank ``g_a``/``g_b`` + # pair, which the shared mixer reads from the same key Kimi K3 uses. + config.linear_attn_config.setdefault("use_full_rank_gate", False) + linear = dict(config.linear_attn_config) + # The mixer's own row-parallel o_proj AllReduce (fixed strategy; the + # size-aware Glm5NextAllReduce used elsewhere is a perf option, not a + # correctness requirement, and AUTO's autotuner raced at TP4 decode). + super().__init__( + config, layer_idx, mapping=mapping, allreduce_strategy=glm5_next_allreduce_strategy() + ) + # The checkpoint publishes ``A_log`` / ``dt_bias`` in fp32 (the mixer + # stores them in bf16 by default; the kernels consume fp32 copies + # either way). Rebuilt without ``.detach``/``.data``, which MetaInitMode + # rejects on meta tensors. + for name in ("A_log", "dt_bias"): + param = getattr(self, name) + data = ( + torch.empty(param.shape, dtype=torch.float32, device=param.device) + if param.is_meta + else param.detach().float() + ) + setattr(self, name, nn.Parameter(data, requires_grad=False)) + self.tp_size = self._kda_tp_size + self.tp_rank = self._kda_tp_rank + self.total_num_heads = int(linear["num_heads"]) + self.qkv_dim = self.proj_size + self.total_qkv_dim = self.total_num_heads * self.head_dim + + # -- tensor-parallel ownership (consumed by the exact-placement loader) -- + + def kda_head_range(self) -> tuple[int, int]: + """This rank's contiguous ``[start, end)`` on the head axis.""" + return self.tp_rank * self.num_heads, (self.tp_rank + 1) * self.num_heads + + def kda_channel_range(self) -> tuple[int, int]: + """This rank's contiguous ``[start, end)`` on the ``heads * head_dim`` axis.""" + start, end = self.kda_head_range() + return start * self.head_dim, end * self.head_dim + + def shard_checkpoint_tensor(self, name: str, tensor: torch.Tensor) -> torch.Tensor: + """Slice one full-width checkpoint tensor (module-relative ``name``) + down to this rank's head range.""" + if self.tp_size == 1 or name in self._REPLICATED: + return tensor + if name in self._CHANNEL_ROW_SHARDED: + start, end = self.kda_channel_range() + return tensor[start:end] + if name in self._HEAD_ROW_SHARDED: + start, end = self.kda_head_range() + return tensor[start:end] + if name == "o_proj.weight": + start, end = self.kda_channel_range() + return tensor[:, start:end] + raise ValueError(f"glm5_next KDA has no tensor-parallel ownership for {name!r}") + + def finalize_weights(self) -> None: + """Post-load fused decode projections and kernel constants, as the + Kimi K3 loader does (the fused shards become the parameters' storage, + so nothing is duplicated).""" + self.finalize_decode_weights() + self._build_mtp_conv_weights() + + +def _glm5_linear_kwargs( + model_config: ModelConfig | None, mapping: Mapping | None +) -> dict[str, Any]: + """Construction arguments shared by every Mapping-aware ``Linear`` here. + + The quant config is the checkpoint's (``FP8_BLOCK_SCALES`` with its + published ``modules_to_not_convert``); the base class's + ``apply_quant_config_exclude_modules`` then flips the excluded modules to + bf16 by *runtime module name* before weights are created, so no + model-specific plan is needed. ``disable_deep_gemm`` pins the block-FP8 + GEMM to ``fp8_quantize_1x128`` + the CuTe DSL Blackwell kernel (the same + arithmetic the routed experts run; DeepGEMM is not available here). + Direct (harness) construction without a ``model_config`` yields plain + bf16 modules over ``mapping``. + """ + return { + "bias": False, + "dtype": torch.bfloat16, + "mapping": glm5_next_attention_mapping( + model_config.mapping if model_config is not None else mapping + ), + "quant_config": model_config.get_quant_config() if model_config is not None else None, + "skip_create_weights_in_init": ( + model_config.skip_create_weights_in_init if model_config is not None else False + ), + "allreduce_strategy": glm5_next_allreduce_strategy(), + "disable_deep_gemm": True, + } + + +class Glm5NextIndexer(nn.Module): + """Pool-compressed DSA indexer (``Glm5NextTextIndexer``). + + Stock DeepSeek-V3.2 DSA selects ``index_topk`` individual keys. This one + selects ``index_topk / index_kpool`` *compressed pools*, expands each back + into its member positions, and always appends the incomplete tail, so the + two are not interchangeable even though both end up with ~2048 positions. + + The backend caches each token's key and compression gate, plus the pooled + key at each pool's first row. It updates the affected pool when tokens are + appended; only complete pools are scored, and the current tail is selected + separately so future tokens cannot influence a query. + + Unlike the HF module there is no left padding here: TensorRT-LLM stores + exactly one request's tokens per cache slot starting at position 0, so + ``first_key`` is always 0 and the packed validity channel HF carries for + padded batches is replaced by the request's own ``kv_len``. + + Tensor parallelism: the whole indexer is replicated (the vLLM/SGLang + ownership) -- every rank runs all 32 scoring heads on the replicated + pool-key path (``wk``, ``k_norm``, the APE and compress gate) and selects + identical pool/tail/-1 indices by identical compute, with no collective. + The two scoring GEMMs are tiny (1536->4096, 4096->32), far cheaper than + the fp32 ``[tokens, pools]`` score all-reduce the sharded-head design + needed before top-k. + + Decode and prefill score cached *pool keys* (the trailing + ``head_dim`` columns of each pool's first-member cache row, maintained by + the backend's ``update_pool_keys``) through the fused paged kernels. + """ + + #: ``wk``, the pool-compress gate and ``weights_proj`` all read the same + #: hidden state and are BF16: the runtime holds them as one fused Linear + #: (``[k | gate | head weights]`` rows, in this order); the loader + #: concatenates the checkpoint's three tensors into it. + FUSED_LINEARS = (("wk_gate_wp", ("wk", "index_kpool_compress_gate", "weights_proj")),) + + def __init__( + self, + config: PretrainedConfig, + layer_idx: int, + dtype: torch.dtype = torch.bfloat16, + mapping: Mapping | None = None, + model_config: ModelConfig | None = None, + ) -> None: + super().__init__() + del dtype # bf16 module (the checkpoint publishes every indexer tensor in bf16) + if model_config is not None: + mapping = model_config.mapping + self.layer_idx = layer_idx + self.hidden_size = int(config.hidden_size) + self.total_n_heads = int(config.index_n_heads) + self.tp_size = int(getattr(mapping, "tp_size", 1) or 1) + self.tp_rank = int(getattr(mapping, "tp_rank", 0) or 0) + if self.total_n_heads % self.tp_size: + raise ValueError( + f"glm5_next indexer has {self.total_n_heads} scoring heads, not " + f"divisible by tp_size {self.tp_size}" + ) + # Replicated scoring heads (vLLM/SGLang ownership): every rank runs + # all 32 heads and selects identical indices by identical compute, + # so no score collective is needed before top-k. + self.n_heads = self.total_n_heads + self.head_dim = int(config.index_head_dim) + self.index_topk = int(config.index_topk) + self.index_kpool = int(config.index_kpool) + self.always_select_tail = bool(config.index_kpool_always_select_tail) + self.softmax_scale = self.head_dim**-0.5 + self.head_mix_scale = self.total_n_heads**-0.5 + self.select_k = self.index_topk // self.index_kpool + self.output_width = self.index_topk + ( + self.index_kpool - 1 if self.always_select_tail else 0 + ) + + # Replicated projections (no tensor_parallel_mode). + lin_kwargs = _glm5_linear_kwargs(model_config, mapping) + self.wq_b = Linear( + int(config.q_lora_rank), self.total_n_heads * self.head_dim, **lin_kwargs + ) + self.wk_gate_wp = Linear( + self.hidden_size, 2 * self.head_dim + self.total_n_heads, **lin_kwargs + ) + self.k_norm = LayerNorm(hidden_size=self.head_dim, eps=1e-6, dtype=torch.bfloat16) + self.index_kpool_compress_ape = nn.Parameter( + torch.zeros(self.index_kpool, self.head_dim, dtype=torch.bfloat16) + ) + # Decode top-k over the fused pool scores: the DSA indexer's TopK + # module. The CuTe-DSL radix kernel is bounded by each request's + # candidate count (1-6 us here vs 20-30 us for the CUDA radix kernel + # and ~40 us for torch.topk over the capacity-wide row); identical + # selections measured. Falls back to the CUDA radix kernel where the + # CUTLASS DSL is unavailable. + from ..cute_dsl_utils import IS_CUTLASS_DSL_AVAILABLE + from ..modules.top_k import TopK, TopKImplementation + + self.pool_top_k = TopK( + self.select_k, + prefill_implementation=TopKImplementation.TORCH, + decode_implementation=( + TopKImplementation.CUTE_DSL_RADIX + if IS_CUTLASS_DSL_AVAILABLE + else TopKImplementation.CUDA_RADIX + ), + ) + + @property + def cache_state_dim(self) -> int: + """Width of one indexer cache row: ``[k | gate | pool key]``. + + The trailing ``head_dim`` columns hold, on a pool's first-member row, + the pool's compressed key -- maintained incrementally by the backend + so decode scores cached pool keys instead of rebuilding every pool + from the packed state each step. + """ + return 3 * self.head_dim + + @property + def packed_state_dim(self) -> int: + """Width of the cached per-token state: ``[k | gate]``.""" + return 2 * self.head_dim + + def project_state(self, hidden_states: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """``(packed [k(head_dim) | gate(head_dim)], head weights [n_heads])`` + from the one fused input GEMM.""" + out = self.wk_gate_wp(hidden_states) + hd = self.head_dim + packed = torch.cat([self.k_norm(out[:, :hd]), out[:, hd : 2 * hd]], dim=-1) + return packed, out[:, 2 * hd :] + + def packed_state(self, hidden_states: torch.Tensor) -> torch.Tensor: + """Per-token ``[k(head_dim) | gate(head_dim)]`` written to the cache.""" + return self.project_state(hidden_states)[0] + + +class Glm5NextSparseAttention(nn.Module): + """Fully NoPE sparse MLA with a pool-compressed indexer. + + ``qk_rope_head_dim`` is 0 on this checkpoint and ``mla_use_nope`` is true: + there is no text rotary call and no rotary cache. ``indexer_rope_interleave`` + is present in the config but vestigial for the text path, so no rotary + branch is created for it -- long-range position sensitivity comes from the + causal KDA layers and the indexer's learned pool APE. + + Layering follows the attention developer guide. This module owns the + module math only: low-rank q/kv projections and norms, the pool indexer's + scoring/selection (model-layer sparse prediction, as in MiniMax-M3), the + absorbed-MLA query/value reassociation, and ``o_proj``. Everything below + that -- the paged latent/indexer cache path and the sparse-MLA core -- + belongs to ``self.attn_backend``, a + :class:`~tensorrt_llm._torch.attention.backends.sparse.glm_kpool.GlmKpoolSparseAttention`: + a ``TrtllmAttention`` subclass (the fully-NoPE branch of the TRTLLM sparse + family) constructed through the standard ``create_attention(...)`` + dispatch on the configured backend slot (``ModelConfig.attn_backend``, + default TRTLLM) with ``SparseParams(algorithm="glm_kpool")``. Its typed + metadata family is ``TrtllmAttentionMetadata`` (``attn_backend.Metadata``), + the class the engine constructs for this model. + + Cache ownership: one latent ``kv_lora_rank``-wide entry per token (the + pre-``kv_b_proj`` latent, not expanded K/V) plus the indexer's packed + ``[k | gate]`` state, both held by the one hybrid ``KVCacheManagerV2`` and + read/written only by the backend, which derives every pool, block table, + and visible length from the prepared attention metadata it is handed -- + this module passes schedule values and the metadata, never raw pools. The + pool-expanded selection travels to the backend inside the standard + ``AttentionForwardArgs.sparse_backend_args`` carrier. + + ``kv_b_proj`` is on this checkpoint's ``modules_to_not_convert`` list + (BF16), so absorption is a reassociation of the same BF16 weights, not a + dequantization. + + Tensor parallelism: with ``tp_size > 1`` this module owns + ``num_heads = 64 // tp_size`` local query heads and the matching rows of + the column-sharded ``q_b_proj``/``kv_b_proj`` (so the absorbed per-head + views are local by construction), while the low-rank latents + (``q_a``/``kv_a``) and both norms stay replicated and the row-sharded + ``o_proj`` returns a partial that this module reduces once through + :class:`Glm5NextAllReduce` -- the DeepSeek-V3 MLA ownership with a + message-size-aware collective. The latent cache and the indexer's packed state stay + *complete* (512- and 256-wide) on every rank: ``num_kv_heads == 1`` is + never divided, all ranks compute identical latent/packed rows from the + replicated projections, and each rank's backend reads its own full copy. + The backend is constructed with the local head count, exactly as MLA's + per-rank ``num_heads // tp_size``. At ``tp_size == 1`` construction and + math are byte-identical to the pre-TP module. + """ + + #: The two low-rank input projections are both block-FP8 and replicated: + #: one fused Linear (``[q_a | kv_a]`` rows), one activation quantization + #: -- the DeepSeek-V3 ``fuse_qkv_a_proj`` layout under the same name. + FUSED_LINEARS = (("kv_a_proj_with_mqa", ("q_a_proj", "kv_a_proj_with_mqa")),) + + def __init__( + self, + config: PretrainedConfig, + layer_idx: int, + dtype: torch.dtype = torch.bfloat16, + attn_backend: str = "TRTLLM", + mapping: Mapping | None = None, + model_config: ModelConfig | None = None, + ) -> None: + super().__init__() + if model_config is not None: + mapping = model_config.mapping + # Heads are sharded over TP, or replicated per rank under attention DP. + attn_mapping = glm5_next_attention_mapping(mapping) + self.layer_idx = layer_idx + self.hidden_size = int(config.hidden_size) + self.total_num_heads = int(config.num_attention_heads) + self.tp_size = int(getattr(attn_mapping, "tp_size", 1) or 1) + self.tp_rank = int(getattr(attn_mapping, "tp_rank", 0) or 0) + if self.total_num_heads % self.tp_size: + raise ValueError( + f"glm5_next sparse MLA has {self.total_num_heads} heads, not divisible " + f"by tp_size {self.tp_size}" + ) + self.num_heads = self.total_num_heads // self.tp_size + self.q_lora_rank = int(config.q_lora_rank) + self.kv_lora_rank = int(config.kv_lora_rank) + self.qk_nope_head_dim = int(config.qk_nope_head_dim) + self.qk_rope_head_dim = int(config.qk_rope_head_dim) + self.v_head_dim = int(config.v_head_dim) + self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim + if self.qk_rope_head_dim != 0: + raise ValueError( + "glm5_next text attention is fully NoPE; a non-zero qk_rope_head_dim " + f"({self.qk_rope_head_dim}) would need a rotary path this bring-up does not have" + ) + self.scaling = self.qk_head_dim**-0.5 + eps = float(config.rms_norm_eps) + + lin_kwargs = _glm5_linear_kwargs(model_config, mapping) + # Low-rank latents replicated, per-head maps column-sharded, output + # row-sharded and returned as a partial (the module runs the branch's + # one reduction through Glm5NextAllReduce). + self.kv_a_proj_with_mqa = Linear( + self.hidden_size, + self.q_lora_rank + self.kv_lora_rank + self.qk_rope_head_dim, + **lin_kwargs, + ) + self.q_a_layernorm = RMSNorm(hidden_size=self.q_lora_rank, eps=eps, dtype=dtype) + self.q_b_proj = Linear( + self.q_lora_rank, + self.total_num_heads * self.qk_head_dim, + tensor_parallel_mode=TensorParallelMode.COLUMN, + **lin_kwargs, + ) + self.kv_a_layernorm = RMSNorm(hidden_size=self.kv_lora_rank, eps=eps, dtype=dtype) + self.kv_b_proj = Linear( + self.kv_lora_rank, + self.total_num_heads * (self.qk_nope_head_dim + self.v_head_dim), + tensor_parallel_mode=TensorParallelMode.COLUMN, + **lin_kwargs, + ) + self.o_proj = Linear( + self.total_num_heads * self.v_head_dim, + self.hidden_size, + tensor_parallel_mode=TensorParallelMode.ROW, + reduce_output=False, + **lin_kwargs, + ) + self.tp_all_reduce = Glm5NextAllReduce(mapping) if glm5_next_tp_reduces(mapping) else None + self.indexer = Glm5NextIndexer( + config, layer_idx, mapping=mapping, model_config=model_config + ) + + # The production sparse-MLA backend, selected through the standard + # dispatch (`get_attention_backend(attn_backend, sparse_params)` via + # `create_attention`). It consumes absorbed latent-space queries, so + # its head_dim is kv_lora_rank, and the latent cache is MQA-style + # (one KV head). AttentionBackend is not an nn.Module: this is a + # plain attribute, invisible to state_dict/loading. + self.attn_backend = create_attention( + attn_backend, + layer_idx, + num_heads=self.num_heads, + head_dim=self.kv_lora_rank, + num_kv_heads=1, + dtype=dtype, + sparse_params=GlmKpoolSparseParams( + kv_lora_rank=self.kv_lora_rank, + qk_nope_head_dim=self.qk_nope_head_dim, + q_lora_rank=self.q_lora_rank, + v_head_dim=self.v_head_dim, + index_topk=self.indexer.index_topk, + index_kpool=self.indexer.index_kpool, + index_always_select_tail=self.indexer.always_select_tail, + index_head_dim=self.indexer.head_dim, + ), + ) + + def project_inputs(self, hidden_states: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: + """``(q_resid [T, q_lora], latent [T, kv_lora])`` from the fused GEMM.""" + qa_kva = self.kv_a_proj_with_mqa(hidden_states) + q_resid = self.q_a_layernorm(qa_kva[:, : self.q_lora_rank]) + latent = self.kv_a_layernorm(qa_kva[:, self.q_lora_rank :]) + return q_resid, latent + + def _decode_projections( + self, hidden_states: torch.Tensor + ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """``(q_resid, latent, packed [k | gate], index_weights)`` -- the two + fused input GEMMs (attention and indexer).""" + q_resid, latent = self.project_inputs(hidden_states) + packed, weights = self.indexer.project_state(hidden_states) + return q_resid, latent, packed, weights + + def absorbed_kv_b(self) -> tuple[torch.Tensor, torch.Tensor]: + """Per-head absorbed views of ``kv_b_proj``: ``(w_k, w_v_t)``. + + ``w_k`` is ``[H, qk_nope, kv_lora]`` (queries -> latent space) and + ``w_v_t`` is ``[H, kv_lora, v_head]`` (latent attention output -> V + space). Score and value absorption are exact reassociations of the + unabsorbed math: ``q . (W_k @ c) == (W_k^T @ q) . c`` and + ``sum_j p_j (W_v @ c_j) == W_v @ sum_j p_j c_j``. Contiguous copies + are cached after weight loading (keyed on the weight's data pointer + and version) so the per-step bmm never re-materializes them; the + cache is a plain tuple attribute, invisible to ``state_dict``. + """ + weight = self.kv_b_proj.weight + key = (weight.data_ptr(), weight._version) + cached = self.__dict__.get("_absorbed_kv_b_cache") + if cached is not None and cached[0] == key: + return cached[1], cached[2] + if weight.dtype != torch.bfloat16: + raise ValueError( + "glm5_next sparse MLA absorption expects the BF16-excluded " + f"kv_b_proj weight, got dtype {weight.dtype}" + ) + per_head = weight.view( + self.num_heads, self.qk_nope_head_dim + self.v_head_dim, self.kv_lora_rank + ) + w_k = per_head[:, : self.qk_nope_head_dim, :].contiguous() + w_v_t = per_head[:, self.qk_nope_head_dim :, :].transpose(1, 2).contiguous() + self._absorbed_kv_b_cache = (key, w_k, w_v_t) + return w_k, w_v_t + + def absorb_query(self, query: torch.Tensor) -> torch.Tensor: + """``[T, H, qk_nope] -> [T, H, kv_lora]`` (fully NoPE: q is all nope).""" + w_k, _ = self.absorbed_kv_b() + return torch.bmm(query.transpose(0, 1), w_k).transpose(0, 1) + + def project_output_latent(self, out_latent: torch.Tensor, reduce: bool = True) -> torch.Tensor: + """Flat latent attention output -> V space -> ``o_proj``. ``[T, hidden]``. + + ``out_latent`` is the backend's base-contract result + ``[T, num_heads * kv_lora]``; the per-head view for the absorbed V + projection is this module's concern, not the backend boundary's. + """ + _, w_v_t = self.absorbed_kv_b() + tokens = out_latent.shape[0] + per_head = out_latent.view(tokens, self.num_heads, self.kv_lora_rank) + out = torch.bmm(per_head.transpose(0, 1), w_v_t) # [H, T, v_head] + out = self.o_proj(out.transpose(0, 1).reshape(tokens, -1)) + if not reduce or self.tp_all_reduce is None: + return out + return self.tp_all_reduce(out) + + def _select_and_attend_paged( + self, + q_resid: torch.Tensor, + query: torch.Tensor, + index_weights: torch.Tensor, + visible: torch.Tensor, + metadata: AttentionMetadata, + input_type: AttentionInputType, + request_index: int | None = None, + rows_per_request: int = 1, + reduce: bool = True, + request_ids: torch.Tensor | None = None, + ) -> torch.Tensor: + """Fused indexer selection over the paged cache, then sparse attention. + + Score the cached pool keys, top-k, expand + tail + row translation, + attend. ``visible[i]`` is query row ``i``'s visible length (its own + position + 1); ``request_index`` selects one context request's block + table, ``request_ids`` (``[rows]`` int32) the block table of each + packed context row (``None`` for both: the generation rows). Every step is a fixed-shape + kernel with work proportional to the visible length, not the buffer + capacity, so decode replays it inside CUDA graphs and prefill no + longer materializes ``[tokens, pools, head_dim]`` gathers. + """ + indexer = self.indexer + rows = q_resid.shape[0] + q_index = indexer.wq_b(q_resid).view(rows, indexer.n_heads, indexer.head_dim) + # Generation rows read their visible length from the metadata unless + # each request contributes several rows (context or verification). + plain_decode = request_index is None and request_ids is None and rows_per_request == 1 + kv_lens = None if plain_decode else visible + scores = self.attn_backend.score_pools( + q_index, + index_weights, + metadata, + q_scale=indexer.softmax_scale, + w_scale=indexer.head_mix_scale, + request_index=request_index, + kv_lens=kv_lens, + rows_per_request=rows_per_request, + request_ids=request_ids, + ) + num_cand = (visible // indexer.index_kpool).to(torch.int32) + selected = torch.empty(rows, indexer.select_k, dtype=torch.int32, device=scores.device) + indexer.pool_top_k( + scores, + selected, + is_prefill=False, + sequence_lengths=num_cand, + scan_lengths=num_cand, + ) + topk_rows = self.attn_backend.expand_selection( + selected, + metadata, + request_index=request_index, + kv_lens=kv_lens, + rows_per_request=rows_per_request, + request_ids=request_ids, + ) + out_latent = self.attn_backend.forward( + self.absorb_query(query), + None, + None, + metadata, + AttentionForwardArgs( + attention_input_type=input_type, + sparse_backend_args=GlmKpoolBackendForwardArgs(topk_rows=topk_rows), + ), + ) + return self.project_output_latent(out_latent, reduce) + + def forward_prefill( + self, + hidden_states: torch.Tensor, + cu_seqlens: Sequence[int], + cached_lens: Sequence[int], + metadata: AttentionMetadata, + reduce: bool = True, + ctx_rows_fn: Callable[[int, torch.device], Glm5NextContextRows] | None = None, + ) -> torch.Tensor: + """Context phase, including continuation chunks -- all requests at once. + + ``cached_lens[i]`` is how many tokens of request ``i`` are already in + the cache, so a chunk is scored against the whole visible prefix rather + than only against its own tokens. Scoring a chunk in isolation is the + classic chunked-prefill bug here: it still passes a one-shot test. + Only schedule values are passed here; every pool write and read goes + through the backend's metadata-derived cache path. + + The packed context tokens of every request go through each stage in + one launch (projections, cache write, pool-key refresh, scoring, + top-k, expansion, attention): the kernels address each row's block + table through :class:`Glm5NextContextRows.request_ids`. A per-request + Python loop here was the dominant cost of mixed context+generation + iterations (measured 50% GPU idle with ~17 short prompts in flight). + ``ctx_rows_fn`` (the runtime context's cache) shares the schedule + across the sparse layers of one forward. + """ + kpool = self.indexer.index_kpool + device = hidden_states.device + if ctx_rows_fn is not None: + rows = ctx_rows_fn(kpool, device) + else: + rows = Glm5NextContextRows.build(cu_seqlens, cached_lens, kpool, device) + q_resid, latent, packed, index_weights = self._decode_projections(hidden_states) + self.attn_backend.append_paged_state( + latent, packed, rows.positions, metadata, request_ids=rows.request_ids + ) + # Pool keys for every pool this forward completes (one program per + # pool: only pool-final positions). A pool left incomplete at a chunk + # end is refreshed by whichever later write completes it. + if rows.final_positions.shape[0]: + self.attn_backend.update_pool_keys( + rows.final_positions, + self.indexer.index_kpool_compress_ape, + metadata, + request_ids=rows.final_request_ids, + ) + query = self.q_b_proj(q_resid).view(-1, self.num_heads, self.qk_head_dim) + return self._select_and_attend_paged( + q_resid, + query, + index_weights, + rows.positions + 1, + metadata, + AttentionInputType.context_only, + reduce=reduce, + request_ids=rows.request_ids, + ) + + def forward_decode( + self, + hidden_states: torch.Tensor, + kv_lens: torch.Tensor, + metadata: AttentionMetadata, + reduce: bool = True, + ) -> torch.Tensor: + """Generation phase: one token per request, fully batched. + + CUDA-graph contract: every shape here is a function of the *buffer* + geometry (the metadata's block-table width times tokens_per_block), + never of the current lengths, and every request-dependent value + (``kv_lens`` and the metadata's block tables) is a device tensor + refreshed by metadata ``prepare()`` outside the captured region. No + ``.item()``/``.tolist()``, no per-request Python loop, no + host->device copy, and no data-dependent branch runs on this path, so + a captured decode graph replays correctly as lengths grow and slots + are reused. + + ``kv_lens[i]`` is the request's visible length *including* the token + being decoded, so the new token's position is ``kv_lens[i] - 1``; it + must be the same prepare()-refreshed lengths the metadata carries, + sliced to the generation rows. Positions at or beyond a request's + ``kv_lens`` gather page-0 garbage in the indexer prefix; the backend + masks them by replacement and the indexer's own validity masks exclude + them from pools, selection, and the tail. The attention core never + gathers the latent at all: the backend reads the paged pool directly + through its storage row view derived from the metadata, and only + selected (valid) positions are translated into row ids -- sentinels + stay ``-1``. There is no empty-row assertion here: it would force a + host sync, which is illegal under CUDA-graph capture -- and a decode + query is always covered by construction, either by the + always-selected tail (``visible % kpool != 0``) or by the final + complete pool, whose last member *is* the query position + (``visible % kpool == 0``). + """ + batch = hidden_states.shape[0] + positions = kv_lens - 1 # [B] + indexer = self.indexer + q_resid, latent, packed, index_weights = self._decode_projections(hidden_states) + self.attn_backend.append_paged_state( + latent.unsqueeze(1), packed.unsqueeze(1), positions.unsqueeze(1), metadata + ) + # Fused indexer path: refresh the pool the new token belongs to, then + # score / top-k / expand over the cached pool keys. + self.attn_backend.update_pool_keys(positions, indexer.index_kpool_compress_ape, metadata) + query = self.q_b_proj(q_resid).view(batch, self.num_heads, self.qk_head_dim) + return self._select_and_attend_paged( + q_resid, + query, + index_weights, + kv_lens, + metadata, + AttentionInputType.generation_only, + reduce=reduce, + ) + + def forward_mixed( + self, + hidden_states: torch.Tensor, + *, + num_ctx_tokens: int, + prefill: dict[str, Any], + decode: dict[str, Any], + ) -> torch.Tensor: + """Context rows through the prefill path, generation rows through the + fused decode path, one TP reduction over the concatenated output.""" + ctx = self.forward_prefill(hidden_states[:num_ctx_tokens], reduce=False, **prefill) + gen = self.forward_decode(hidden_states[num_ctx_tokens:], reduce=False, **decode) + out = torch.cat([ctx, gen], dim=0) + return out if self.tp_all_reduce is None else self.tp_all_reduce(out) + + def forward_verify( + self, + hidden_states: torch.Tensor, + kv_lens: torch.Tensor, + metadata: AttentionMetadata, + tokens_per_request: int, + ) -> torch.Tensor: + """Generation phase with ``tokens_per_request`` tokens per request. + + The speculative-decoding target scores the golden token plus the + drafts in one pass; the MTP draft layer's first step sees the same + packed layout. Request ``i``'s tokens sit at cache positions + ``kv_lens[i] - T .. kv_lens[i] - 1`` (``kv_lens`` already counts them, + exactly as in :meth:`forward_decode`), so the latent/indexer rows are + appended there and every query is scored against its *own* visible + prefix: pool scoring masks pools whose last member lies beyond the + query's position, and selection expansion builds the tail from the query's + own position, so token ``j`` never sees tokens ``j+1..``. Rows written + for drafts that are later rejected are simply overwritten by the next + step, which re-appends at the rewound ``kv_lens`` -- the same + positional-cache convention MLA relies on. + + Every shape is a function of ``tokens_per_request`` and the buffer + geometry, with no host sync, so a captured verification graph replays + against refreshed lengths and tables. + """ + tokens_per_request = int(tokens_per_request) + if tokens_per_request <= 1: + return self.forward_decode(hidden_states, kv_lens, metadata) + num_tokens = hidden_states.shape[0] + batch = num_tokens // tokens_per_request + if batch * tokens_per_request != num_tokens or kv_lens.shape[0] != batch: + raise ValueError( + f"glm5_next sparse verify: {num_tokens} tokens for {kv_lens.shape[0]} requests " + f"at {tokens_per_request} tokens per request" + ) + device = hidden_states.device + steps = torch.arange(tokens_per_request, device=device) + positions = (kv_lens - tokens_per_request).unsqueeze(1) + steps # [B, T] + q_resid, latent, packed, index_weights = self._decode_projections(hidden_states) + self.attn_backend.append_paged_state( + latent.view(batch, tokens_per_request, -1), + packed.view(batch, tokens_per_request, -1), + positions, + metadata, + ) + # Keep the cached pool keys current, in position order, so a pool + # spanning several drafts ends up with all of its visible members. + for step in range(tokens_per_request): + self.attn_backend.update_pool_keys( + positions[:, step].contiguous(), self.indexer.index_kpool_compress_ape, metadata + ) + query = self.q_b_proj(q_resid).view(num_tokens, self.num_heads, self.qk_head_dim) + # Row i of request b sees its own prefix: visible = position + 1. + return self._select_and_attend_paged( + q_resid, + query, + index_weights, + (positions + 1).reshape(-1), + metadata, + AttentionInputType.generation_only, + rows_per_request=tokens_per_request, + ) + + +# --------------------------------------------------------------------------- +# Heterogeneous request state +# --------------------------------------------------------------------------- + + +def glm5_next_mamba_metadata_cls() -> type[Mamba2Metadata]: + """Return the model's ``Mamba2Metadata`` subclass (see the sparse backend's ``cache_manager``).""" + return Glm5NextMamba2Metadata + + +def glm5_next_cache_manager_cls() -> type[MambaHybridCacheManagerV2]: + """Return the model's ``KVCacheManagerV2`` subclass (see the sparse backend's ``cache_manager``).""" + return Glm5NextCacheManager + + +class Glm5NextGate(DeepseekV3Gate): + """The shared DeepSeek noaux_tc gate, kept FP32 end to end. + + Same routing math and weights as :class:`DeepseekV3Gate` (selection on + bias-corrected sigmoid scores, weights gathered from the uncorrected + scores, normalization before ``routed_scaling_factor``); the two + overrides are the parameter dtypes and the logits GEMM. The correction + bias sits around magnitude ~10 while the sigmoid scores it corrects are + O(1e-2), so ranking turns on inter-expert gaps of 4e-5 - 6e-4; bf16 + resolution at that magnitude is ~1e-2, three orders of magnitude too + coarse -- it silently changes the top-8 while every aggregate check still + passes. The router weight is FP32 in the checkpoint, so the logits are an + FP32 ``F.linear`` (the shared bf16 router GEMM does not apply). + """ + + def __init__(self, config: PretrainedConfig, moe_backend: str = "CUTLASS") -> None: + if str(getattr(config, "scoring_func", "sigmoid")) != "sigmoid": + raise ValueError(f"glm5_next expects sigmoid scoring, got {config.scoring_func!r}") + super().__init__( + int(config.hidden_size), + int(config.n_routed_experts), + top_k=int(config.num_experts_per_tok), + n_group=int(getattr(config, "n_group", 1) or 1), + topk_group=int(getattr(config, "topk_group", 1) or 1), + routed_scaling_factor=float(config.routed_scaling_factor), + dtype=torch.float32, + fuse_routing_kernel=True, + apply_routing=False, + moe_backend=moe_backend, + ) + self.hidden_size = int(config.hidden_size) + self.norm_topk_prob = bool(config.norm_topk_prob) + # FP32 regardless of the MoE backend (the base class picks bf16 for + # TRTLLM); the fused routing kernels consume the FP32 tensor directly. + self.e_score_correction_bias = nn.Parameter( + torch.empty(int(config.n_routed_experts), dtype=torch.float32), requires_grad=False + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + flat = hidden_states.reshape(-1, self.hidden_size) + return torch.nn.functional.linear(flat.float(), self.weight) + + +class Glm5NextMoE(nn.Module): + """Routed experts plus one always-active shared expert. + + The routed experts are a fused-MoE layer built through ``create_moe`` -- + the one selection entry point -- with the DeepSeek noaux_tc routing method + (:class:`Glm5NextGate`, the shared DeepSeek gate kept FP32) and the + DSV4-style uniform ``swiglu_limit_scalar``. On this checkpoint (FP8 block + scales, SM100) the resolver lands on ``TRTLLMGenFusedMoE`` whose + ``trtllm::fp8_block_scale_moe_runner`` consumes the clamp limit as + ``gemm1_clamp_limit``; the ``AUTO`` ``moe_backend`` default resolves to + ``TRTLLM`` for exactly this quant/SM pair in + ``ModelConfig.resolve_moe_backend``. The shared expert is the shared + ``GatedMLP`` (DeepSeek-V3 composition): TP-sharded with + ``reduce_output=False``, its partial summed with the routed partial before + this module's single all-reduce. + """ + + def __init__( + self, + config: PretrainedConfig, + model_config: ModelConfig, + layer_idx: int, + dtype: torch.dtype = torch.bfloat16, + ) -> None: + super().__init__() + hidden = int(config.hidden_size) + self.hidden_size = hidden + self.moe_intermediate_size = int(config.moe_intermediate_size) + self.num_experts = int(config.n_routed_experts) + self.swiglu_limit = float(config.swiglu_limit) + self.gate = Glm5NextGate(config, moe_backend=str(model_config.moe_backend or "CUTLASS")) + inter = self.moe_intermediate_size + self.moe_backend_name: str | None = None + # The fused routing kernel always normalizes the gathered top-8 + # weights. + if not self.gate.norm_topk_prob: + raise ValueError( + "glm5_next production MoE requires norm_topk_prob=True " + "(the DeepSeek noaux_tc routing kernel always normalizes)" + ) + from ..moe.fused_moe.activation import SwigluActivation + from ..moe.fused_moe.create_moe import create_moe + + # The gate's own routing method (DeepSeek noaux_tc; the bias is + # fetched per call so it follows the parameter through + # to_empty/materialization onto its final device). + routing = self.gate.routing_method + # The experts are uniformly block-FP8 on the published checkpoint; + # the exclusion patterns concern *other* modules (applied by name by + # the base class), so the layer-scoped config the fused backend sees + # carries only the algorithm and block size. A bf16 build (no quant + # config) gets bf16 experts. + quant = model_config.get_quant_config() + experts_quant = ( + QuantConfig(quant_algo=quant.quant_algo, group_size=quant.group_size) + if quant is not None and quant.quant_algo is not None + else None + ) + self.experts = create_moe( + routing_method=routing, + num_experts=self.num_experts, + hidden_size=hidden, + intermediate_size=inter, + dtype=dtype, + reduce_results=False, + model_config=model_config, + override_quant_config=experts_quant, + layer_idx=layer_idx, + activation=SwigluActivation(clamp=self.swiglu_limit), + ) + backend = getattr(self.experts, "backend", self.experts) + self.moe_backend_name = type(backend).__name__ + # Four-rank composition: the fused layer runs + # with reduce_results=False, so its output is a rank partial -- + # a K-dim partial per expert in the TP4 layout (moe_tp_size=4), + # a local-expert partial sum in the TP4/EP4 layout (moe_ep_size=4) + # -- and this module owns the exactly-one reduction that combines + # it together with the TP-sharded shared-expert partial (the + # DeepSeek-V3 composition). + self.mapping = model_config.mapping + self.use_dp = bool(self.mapping.enable_attention_dp) + # Reduce routed and TP-sharded shared-expert partials once. Under + # attention DP the fused layer combines across ranks itself and the + # shared expert is replicated, so there is nothing to reduce here. + self.moe_all_reduce = ( + Glm5NextAllReduce(self.mapping) if glm5_next_tp_reduces(self.mapping) else None + ) + # Shared expert: TP-sharded over ``model_config.mapping`` with + # ``reduce_output=False``, so its output is a rank partial summed with + # the routed partial before this module's single all-reduce + # (replicated, ``overridden_tp_size=1``, under attention DP). + self.shared_experts = GatedMLP( + hidden_size=hidden, + intermediate_size=self.moe_intermediate_size * int(config.n_shared_experts), + bias=False, + dtype=dtype, + config=model_config, + overridden_tp_size=1 if self.use_dp else None, + reduce_output=False, + layer_idx=layer_idx, + is_shared_expert=True, + swiglu_limit=self.swiglu_limit, + # Same GEMM the rest of the model's block-FP8 projections run + # (fp8_quantize_1x128 + the CuTe DSL Blackwell kernel). + disable_deep_gemm=True, + ) + + def forward( + self, x: torch.Tensor, all_rank_num_tokens: list[int] | None = None + ) -> torch.Tensor: + flat = x.reshape(-1, self.hidden_size) + # The fused layer routes internally from the FP32 logits and runs + # FC1 -> clamped SwiGLU -> FC2 in one backend call; one code path + # serves prefill and decode with no host-dependent branching, so + # decode stays CUDA-graph-capturable. ``all_rank_num_tokens`` drives + # the fused layer's dispatch/combine under attention DP. + routed = self.experts( + flat, + self.gate(flat), + all_rank_num_tokens=all_rank_num_tokens if self.use_dp else None, + ) + # Routed and shared are both rank partials (a K-dim partial per expert + # in the TP4 layout, the local-expert partial sum in the TP4/EP4 + # layout). Sum them, then exactly one reduction covers the whole MoE + # branch -- the DeepSeek-V3 order. + mixed = routed + self.shared_experts(flat) + if self.moe_all_reduce is not None: + mixed = self.moe_all_reduce(mixed) + return mixed.view_as(x) + + +# --------------------------------------------------------------------------- +# Hyper-connected decoder +# --------------------------------------------------------------------------- + + +def glm5_next_hyper_connection( + config: PretrainedConfig, dtype: torch.dtype = torch.bfloat16 +) -> mHC: + """Build the shared ``mHC`` for one hyper-connection site. + + This reuses TensorRT-LLM's existing manifold-constrained hyper-connection + rather than adding a model-local one. Two settings are model-specific and + were pinned by measurement against the source module, not by assumption: + + * ``post_mult_value=2.0`` -- the source computes ``2 * sigmoid(...)`` for the + block-output placement weights. Leaving the default 1.0 halves them and + shows up as ``max_abs`` 0.96 on ``post`` against a [0.26, 1.92] range. + * ``sinkhorn_iters`` is the config's own ``hc_sinkhorn_iters`` (20), not + ``iters - 1``. The source runs one initial column normalization plus + ``iters - 1`` row/column rounds, which is what this kernel's ``iters`` + counts; passing 19 leaves a measurable 1.9e-5 residual on ``comb`` versus + 1.3e-6 at 20. + + ``norm_eps`` is the decoder's ``rms_norm_eps`` because the source's + hyper-connection input norm is constructed with it, while ``eps`` and + ``sinkhorn_eps`` are the separate ``hc_eps``. + """ + from ..modules.mhc.hyper_connection import mHC + + return mHC( + mult=int(config.hc_mult), + hidden_size=int(config.hidden_size), + sinkhorn_iters=int(config.hc_sinkhorn_iters), + dtype=dtype, + eps=float(config.hc_eps), + norm_eps=float(config.rms_norm_eps), + sinkhorn_eps=float(config.hc_eps), + post_mult_value=2.0, + ) + + +def glm5_next_expand_streams(embeds: torch.Tensor, hc_mult: int) -> torch.Tensor: + """``[tokens, hidden]`` -> ``[tokens, hc_mult, hidden]``. + + The streams start as exact copies of the embedding and only diverge once the + first hyper-connection mixes them. ``contiguous()`` is required, not merely + tidy: an expanded view aliases a single row, and ``post_mapping`` writes each + stream separately. + """ + return embeds.unsqueeze(-2).expand(-1, hc_mult, -1).contiguous() + + +def glm5_next_hyper_head(hidden_streams: torch.Tensor) -> torch.Tensor: + """Collapse the ``hc_mult`` streams with an **unweighted** mean. + + Deliberately not ``modules.mhc.HCHead``: that head is the DeepSeek-V4 + variant and carries learned ``fn``/``base``/``scale`` weights. GLM-5.3-Flash + has no such parameters in the checkpoint and the source head is a plain + mean, so using the weighted head would require inventing weights. + """ + return hidden_streams.mean(dim=-2) + + +class Glm5NextDecoderLayer(DecoderLayer): + """One decoder layer: two hyper-connection sites wrapping attention and FFN. + + The residual path is *not* an ordinary add. Each site collapses the four + streams into one sequence with the learned ``pre`` weights, runs the + sublayer, then writes the result back across the streams as + ``post * out + comb^T @ residual``. Both module choices come from the two + literal per-layer lists, never from a cadence or ``first_k_dense_replace``. + + Two entry points share one implementation: the runtime ``forward`` + receives ``AttentionMetadata`` (plus the once-per-forward + :class:`Glm5NextRuntimeContext`) and derives this layer's cache + arguments; ``forward_direct`` takes them explicitly and is what the + component tests call. The runtime path is + a thin argument-derivation shim over the direct path, so parity between + them is an argument-sourcing check, not a second implementation. + """ + + def __init__( + self, + config: PretrainedConfig, + layer_idx: int, + schedule: Glm5NextSchedule, + model_config: ModelConfig, + dtype: torch.dtype = torch.bfloat16, + ) -> None: + super().__init__() + self.layer_idx = layer_idx + self.attention_type = schedule.attention[layer_idx] + self.mlp_type = schedule.mlp[layer_idx] + eps = float(config.rms_norm_eps) + + mapping = model_config.mapping + if self.attention_type == LINEAR_ATTENTION: + self.self_attn: nn.Module = Glm5NextLinearAttention( + config, layer_idx, dtype=dtype, mapping=mapping + ) + else: + self.self_attn = Glm5NextSparseAttention( + config, + layer_idx, + dtype=dtype, + attn_backend=model_config.attn_backend, + model_config=model_config, + ) + self.mlp = ( + Glm5NextMoE(config, model_config, layer_idx, dtype=dtype) + if self.mlp_type == SPARSE_MLP + else GatedMLP( + hidden_size=int(config.hidden_size), + intermediate_size=int(config.intermediate_size), + bias=False, + dtype=dtype, + config=model_config, + # Replicated under attention DP (the MLP follows the attention + # layout, as in DeepSeek-V3). + overridden_tp_size=1 if mapping.enable_attention_dp else None, + # The layer owns the branch's single reduction (size-aware + # strategy), as it does for attention and the MoE combine. + reduce_output=False, + layer_idx=layer_idx, + swiglu_limit=float(config.swiglu_limit), + disable_deep_gemm=True, + ) + ) + self.mlp_all_reduce = ( + Glm5NextAllReduce(mapping) + if self.mlp_type != SPARSE_MLP and glm5_next_tp_reduces(mapping) + else None + ) + self.input_layernorm = RMSNorm(hidden_size=int(config.hidden_size), eps=eps, dtype=dtype) + self.post_attention_layernorm = RMSNorm( + hidden_size=int(config.hidden_size), eps=eps, dtype=dtype + ) + self.hc_attn = glm5_next_hyper_connection(config, dtype=dtype) + self.hc_ffn = glm5_next_hyper_connection(config, dtype=dtype) + + def forward( + self, + position_ids: torch.Tensor | None = None, + hidden_states: torch.Tensor | None = None, + attn_metadata: AttentionMetadata | None = None, + runtime_ctx: Glm5NextRuntimeContext | None = None, + **kwargs: Any, + ) -> torch.Tensor: + """Runtime entry: derive this layer's cache arguments from metadata. + + The executor packs context requests first, then generation requests; + the two phases run through :meth:`forward_direct` separately, which is + exact because every non-attention operation here is token-local. + ``position_ids`` is unused: the text path is fully NoPE and the + indexer derives its positions from the cached lengths. + """ + if runtime_ctx is None: + runtime_ctx = build_glm5_next_runtime_context(attn_metadata) + all_rank_num_tokens = getattr(runtime_ctx.metadata, "all_rank_num_tokens", None) + if self.attention_type == LINEAR_ATTENTION: + # The shared KDA mixer splits context / generation rows (prefill, + # decode, verify) itself from the prepared metadata. + return self.forward_direct( + hidden_states, + all_rank_num_tokens=all_rank_num_tokens, + attn_metadata=runtime_ctx.metadata, + ) + derive = runtime_ctx.sparse_kwargs + if runtime_ctx.num_contexts > 0 and runtime_ctx.num_generations > 0: + if runtime_ctx.gen_phase == "decode": + # Mixed context+generation batch: one pass over all tokens + # (hyper-connections, norms, MoE, the o_proj reduction), with + # the attention module splitting the rows internally between + # its prefill and decode kernels -- the vLLM layout. A second + # small-batch pass per layer would cost a whole decode step + # per mixed iteration. + return self.forward_direct( + hidden_states, + phase="mixed", + all_rank_num_tokens=all_rank_num_tokens, + **runtime_ctx.mixed_kwargs(self.layer_idx), + ) + # Speculative verification rows run the replay verify kernel; they + # keep the split path (prefill rows, then verify rows). + parts = [ + self.forward_direct( + hidden_states[: runtime_ctx.num_ctx_tokens], + phase="prefill", + all_rank_num_tokens=all_rank_num_tokens, + **derive(self.layer_idx, "prefill"), + ), + self.forward_direct( + hidden_states[runtime_ctx.num_ctx_tokens :], + phase="verify", + all_rank_num_tokens=all_rank_num_tokens, + **derive(self.layer_idx, "verify"), + ), + ] + return torch.cat(parts, dim=0) + if runtime_ctx.num_contexts > 0: + return self.forward_direct( + hidden_states, + phase="prefill", + all_rank_num_tokens=all_rank_num_tokens, + **derive(self.layer_idx, "prefill"), + ) + # "decode" (one token per request) or "verify" (golden + drafts per + # request while a speculative-decoding target scores them). + phase = runtime_ctx.gen_phase + return self.forward_direct( + hidden_states, + phase=phase, + all_rank_num_tokens=all_rank_num_tokens, + **derive(self.layer_idx, phase), + ) + + def forward_direct( + self, + hidden_streams: torch.Tensor, + phase: str = "prefill", + all_rank_num_tokens: list[int] | None = None, + **attn_kwargs: Any, + ) -> torch.Tensor: + """``hidden_streams`` is ``[num_tokens, hc_mult, hidden]``. + + A KDA layer takes ``attn_metadata`` and runs the shared mixer over all + rows; a sparse layer's ``attn_kwargs`` are forwarded verbatim to its + ``forward_``. ``all_rank_num_tokens`` (attention DP) reaches the + fused MoE. + """ + residual = hidden_streams + post, comb, collapsed = self.hc_attn.pre_mapping(hidden_streams) + normed = self.input_layernorm(collapsed) + if self.attention_type == LINEAR_ATTENTION: + attn_out = self.self_attn(normed, attn_kwargs["attn_metadata"]) + else: + attn_out = getattr(self.self_attn, f"forward_{phase}")(normed, **attn_kwargs) + hidden_streams = self.hc_attn.post_mapping(attn_out, residual, post, comb) + + residual = hidden_streams + post, comb, collapsed = self.hc_ffn.pre_mapping(hidden_streams) + mlp_out = self.run_mlp(self.post_attention_layernorm(collapsed), all_rank_num_tokens) + return self.hc_ffn.post_mapping(mlp_out, residual, post, comb) + + def run_mlp(self, x: torch.Tensor, all_rank_num_tokens: list[int] | None) -> torch.Tensor: + """The FFN branch: the MoE (fed the attention-DP token counts), or + the shared dense ``GatedMLP`` followed by this layer's single TP + reduction of its partial.""" + if self.mlp_type == SPARSE_MLP: + return self.mlp(x, all_rank_num_tokens) + out = self.mlp(x) + return out if self.mlp_all_reduce is None else self.mlp_all_reduce(out) + + +class Glm5NextModel(DecoderModel): + """Embedding, the 45 hyper-connected decoder layers, and the final readout. + + A ``DecoderModel`` whose hidden state between layers is the four-stream + tensor ``[num_tokens, hc_mult, hidden]`` rather than ``[num_tokens, + hidden]``: the stream axis opens at the embedding and closes in + :meth:`collapse_streams` (unweighted mean plus the final norm), which is + this model's equivalent of the base class's trailing ``self.norm``. + """ + + def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: + _normalize_glm5_next_top_config(model_config.pretrained_config) + super().__init__(model_config) + config = get_glm5_next_text_config(model_config.pretrained_config) + schedule = resolve_glm5_next_schedule(model_config.pretrained_config) + # Pipeline parallelism rides the base machinery wholesale: the + # inter-layer activation is the four-stream tensor [tokens, hc_mult, + # hidden], and both `forward_after_recv`/`forward_before_send` and + # `pp_recv/send` are shape-agnostic — the recv buffer on a non-first + # rank is exactly `expand_streams(embed_tokens.skip_forward(...))`, + # which is a real contiguous [tokens, hc_mult, hidden] tensor. + # `__pp_init__` prunes non-local layers; `load_weights` skips + # pruned owners (see `skipped_remote`); the hybrid cache manager + # slices its layer masks per rank from `mapping`. + self.config = config + self.schedule = schedule + self.hc_mult = int(config.hc_mult) + dtype = getattr(config, "torch_dtype", None) or torch.bfloat16 + + self.embed_tokens = Embedding(int(config.vocab_size), int(config.hidden_size), dtype=dtype) + self.layers = nn.ModuleList( + [ + Glm5NextDecoderLayer(config, i, schedule, model_config, dtype=dtype) + for i in range(schedule.num_layers) + ] + ) + self.norm = RMSNorm( + hidden_size=int(config.hidden_size), eps=float(config.rms_norm_eps), dtype=dtype + ) + # Read by the one-model MTP drafter factory (``MTPForCausalLM`` passes + # ``model.aux_stream_dict`` to every MTP layer). This model runs its + # branches on the main stream, so the draft layer receives an empty map. + self.aux_stream_dict: dict[Any, Any] = {} + + def forward( + self, + attn_metadata: AttentionMetadata, + input_ids: torch.Tensor | None = None, + position_ids: torch.Tensor | None = None, + inputs_embeds: torch.Tensor | None = None, + runtime_ctx: Glm5NextRuntimeContext | None = None, + **kwargs: Any, + ) -> torch.Tensor: + """The runtime forward over the executor's packed token batch. + + ``runtime_ctx`` is normally derived from ``attn_metadata`` here; the + parameter lets a caller (or a parity test) inject a context built by + another route. The stream axis opens once, is carried through all 45 + layers, and closes in :meth:`collapse_streams`. + """ + if (input_ids is None) == (inputs_embeds is None): + raise ValueError("specify exactly one of input_ids or inputs_embeds") + if inputs_embeds is None: + inputs_embeds = self.embed_tokens(input_ids) + streams = self.expand_streams(inputs_embeds) + if runtime_ctx is None: + runtime_ctx = build_glm5_next_runtime_context(attn_metadata) + # Only the decoder stack: under one-model MTP the speculative base + # class appends the draft layer(s) to ``self.layers`` (so the + # checkpoint's ``model.layers.45.*`` names resolve), and those run in + # the speculative worker's draft loop, not here. + # Generation-only batches take the fused hyper-connection loop. Prefill + # (and mixed) batches keep the per-layer path so the engine's warmup / + # KV-cache sizing pass measures the same activation peak that mixed + # context+generation iterations reach in production (a lower + # pure-prefill peak over-allocates the KV cache and OOMs later). + if ( + runtime_ctx.num_contexts == 0 + and runtime_ctx.num_generations > 0 + and self._fused_hc_ok() + ): + return self.collapse_streams(self._forward_single_phase_fused_hc(streams, runtime_ctx)) + for layer_idx in range(self.schedule.num_layers): + streams = self.layers[layer_idx]( + position_ids=position_ids, + hidden_states=streams, + attn_metadata=attn_metadata, + runtime_ctx=runtime_ctx, + ) + return self.collapse_streams(streams) + + def _fused_hc_ok(self) -> bool: + """Whether every decoder layer is local (no PP pruning) for the fused loop.""" + cached = self.__dict__.get("_fused_hc_ok_cache") + if cached is None: + cached = all( + not getattr(self.layers[i], "_weights_removed", False) + for i in range(self.schedule.num_layers) + ) + self._fused_hc_ok_cache = cached + return cached + + def _forward_single_phase_fused_hc( + self, streams: torch.Tensor, runtime_ctx: Glm5NextRuntimeContext + ) -> torch.Tensor: + """Single-phase forward with fused hyper-connection boundaries. + + Every boundary between two sublayers is ``post_mapping`` of the + previous site, ``pre_mapping`` of the next, and the next sublayer's + RMSNorm; ``mHC.fused_hc`` (the in-tree DeepSeek-V4 boundary op) runs + the three as one kernel, so a layer costs 2 fused boundaries instead + of 4 mappings + 2 norms. Same math as :meth:`Glm5NextDecoderLayer. + forward_direct` chained over the stack; the first pre-mapping and the + last post-mapping stay unfused. Only generation-only batches use it + (see :meth:`forward`); the phase is still derived here so the loop + stays valid for a pure-context batch. + """ + phase = "prefill" if runtime_ctx.num_generations == 0 else runtime_ctx.gen_phase + num_layers = self.schedule.num_layers + first = self.layers[0] + residual = streams + post, comb, x = first.hc_attn.pre_mapping(streams) + x = first.input_layernorm(x) + for layer_idx in range(num_layers): + layer = self.layers[layer_idx] + if layer.attention_type == LINEAR_ATTENTION: + attn_out = layer.self_attn(x, runtime_ctx.metadata) + else: + attn_out = getattr(layer.self_attn, f"forward_{phase}")( + x, **runtime_ctx.sparse_kwargs(layer_idx, phase) + ) + residual, post, comb, x = layer.hc_ffn.fused_hc( + attn_out, + residual, + post, + comb, + norm_weight=layer.post_attention_layernorm.weight, + norm_eps=layer.post_attention_layernorm.variance_epsilon, + ) + mlp_out = layer.run_mlp(x, getattr(runtime_ctx.metadata, "all_rank_num_tokens", None)) + if layer_idx + 1 < num_layers: + nxt = self.layers[layer_idx + 1] + residual, post, comb, x = nxt.hc_attn.fused_hc( + mlp_out, + residual, + post, + comb, + norm_weight=nxt.input_layernorm.weight, + norm_eps=nxt.input_layernorm.variance_epsilon, + ) + else: + residual = layer.hc_ffn.post_mapping(mlp_out, residual, post, comb) + return residual + + def expand_streams(self, embeds: torch.Tensor) -> torch.Tensor: + """Open the four-stream axis at the embedding.""" + return glm5_next_expand_streams(embeds, self.hc_mult) + + def collapse_streams(self, hidden_streams: torch.Tensor) -> torch.Tensor: + """Unweighted stream mean followed by the final RMS norm.""" + return self.norm(glm5_next_hyper_head(hidden_streams)) + + +# --------------------------------------------------------------------------- +# Multi-token prediction (one-model MTP speculative decoding) +# --------------------------------------------------------------------------- + + +class Glm5NextMTPHead(nn.Module): + """The MTP layer's ``shared_head``: its final norm plus the draft logits. + + ``norm`` is applied by :class:`Glm5NextMTP` at the end of its forward (the + checkpoint's ``shared_head.norm``); ``forward`` here turns the resulting + hidden states into logits through the *target's* ``lm_head`` -- the + checkpoint publishes no separate MTP head weight -- the way the + speculative worker calls it: ``shared_head(hidden, lm_head, attn_metadata, + return_context_logits)``. The LM-head TP handling mirrors the DeepSeek-V3 + MTP head exactly, because the worker's greedy draft sampler recovers the + global argmax from vocab-sharded logits (``is_spec_decoding_head``). + """ + + def __init__( + self, config: PretrainedConfig, model_config: ModelConfig, dtype: torch.dtype + ) -> None: + super().__init__() + self.model_config = model_config + self.norm = RMSNorm( + hidden_size=int(config.hidden_size), eps=float(config.rms_norm_eps), dtype=dtype + ) + + @staticmethod + def get_last_token_states(hidden_states: torch.Tensor, attn_metadata) -> torch.Tensor: + last_tokens = torch.cumsum(attn_metadata.seq_lens_cuda, dim=0, dtype=torch.long) - 1 + return hidden_states[last_tokens] + + def forward( + self, + hidden_states: torch.Tensor, + lm_head: nn.Module, + attn_metadata, + return_context_logits: bool = False, + ) -> torch.Tensor: + if not return_context_logits: + if attn_metadata is not None: + hidden_states = self.get_last_token_states(hidden_states, attn_metadata) + else: + hidden_states = hidden_states[-1].unsqueeze(0) + + # The draft sampler consumes vocab-sharded logits. Preserve the target + # head's setting even if projection fails. + gather_output = lm_head.gather_output + lm_head.gather_output = False + try: + return lm_head(hidden_states, is_spec_decoding_head=True) + finally: + lm_head.gather_output = gather_output + + +class Glm5NextMTP(nn.Module): + """GLM-5.3-Flash's one next-token-prediction layer (``layers.45``). + + Structure, verified against the checkpoint's own keys and the vLLM GLM-5 + port (``glm5next/nvidia/mtp.py``) since the HF reference implements no + MTP: ``enorm(embed(next_token)) ++ hnorm(target_hidden) -> eh_proj`` into + a **plain-residual** decoder block -- the MTP layer has no + hyper-connection weights, unlike the 45 main layers -- made of the same + sparse-MLA + k-pool indexer attention and 288+1-expert MoE as the main + sparse layers, closed by ``shared_head.norm``. The attention and MoE are + the *same classes* as the main stack (with ``layer_idx = 45``), so the + draft layer owns its own latent/indexer pages in the one hybrid cache + manager (the executor appends one attention layer for it) and shares the + projection-swap, TP-shard, and exact-placement loading contracts. + + Constructed by :class:`~.modeling_speculative.MTPForCausalLM` through + the model-type dispatch and called by the one-model speculative worker + as ``mtp_layer(input_ids, position_ids, hidden_states, embed_tokens, + attn_metadata, all_rank_num_tokens, spec_metadata)``. ``hidden_states`` + are the target's post-final-norm states for the same packed tokens (the + convention the DeepSeek-V3/Qwen3-Next/vLLM MTP paths all use), and the + first draft step's token schedule is *identical* to the target + verification pass (contexts: prompt shifted by one; generation: + ``1 + runtime_draft_len`` accepted/padded tokens at the same positions), + so it reuses the target's runtime-context derivation, reading the + metadata's live ``kv_lens_cuda`` so later draft steps' rewinds are seen. + """ + + def __init__( + self, + model_config: ModelConfig[PretrainedConfig], + layer_idx: int, + aux_stream_dict: Any = None, + is_separate_draft_engine: bool = False, + ) -> None: + super().__init__() + del aux_stream_dict # single-stream model; accepted for the factory's call shape + if is_separate_draft_engine: + raise NotImplementedError( + "glm5_next MTP runs one-model speculative decoding only (MTP / MTP_EAGLE_ONE_MODEL)" + ) + _normalize_glm5_next_top_config(model_config.pretrained_config) + config = get_glm5_next_text_config(model_config.pretrained_config) + schedule = resolve_glm5_next_schedule(model_config.pretrained_config) + num_nextn = int(getattr(config, "num_nextn_predict_layers", 0) or 0) + if not (schedule.num_layers <= layer_idx < schedule.num_layers + num_nextn): + raise ValueError( + f"glm5_next MTP layer index {layer_idx} is outside the checkpoint's appended " + f"range [{schedule.num_layers}, {schedule.num_layers + num_nextn})" + ) + dtype = getattr(config, "torch_dtype", None) or torch.bfloat16 + hidden = int(config.hidden_size) + eps = float(config.rms_norm_eps) + self.model_config = model_config + self.layer_idx = layer_idx + self.attention_type = SPARSE_ATTENTION + self.mlp_type = SPARSE_MLP + + self.enorm = RMSNorm(hidden_size=hidden, eps=eps, dtype=dtype) + self.hnorm = RMSNorm(hidden_size=hidden, eps=eps, dtype=dtype) + # Row-parallel (DeepSeek-V3 MTP ownership): each rank consumes its own + # input chunk, one in-Linear reduction; replicated under attention DP. + self.eh_proj = Linear( + 2 * hidden, + hidden, + tensor_parallel_mode=( + None if model_config.mapping.enable_attention_dp else TensorParallelMode.ROW + ), + **_glm5_linear_kwargs(model_config, None), + ) + self.input_layernorm = RMSNorm(hidden_size=hidden, eps=eps, dtype=dtype) + self.post_attention_layernorm = RMSNorm(hidden_size=hidden, eps=eps, dtype=dtype) + self.self_attn = Glm5NextSparseAttention( + config, + layer_idx, + dtype=dtype, + attn_backend=model_config.attn_backend, + model_config=model_config, + ) + self.mlp = Glm5NextMoE(config, model_config, layer_idx, dtype=dtype) + self.shared_head = Glm5NextMTPHead(config, model_config, dtype) + + def forward( + self, + input_ids: torch.Tensor, + position_ids: torch.Tensor | None, + hidden_states: torch.Tensor, + embed_tokens: nn.Module, + attn_metadata: AttentionMetadata, + all_rank_num_tokens: list[int] | None = None, + spec_metadata: Any = None, + **kwargs: Any, + ) -> torch.Tensor: + del position_ids, kwargs # NoPE + if all_rank_num_tokens is None: + all_rank_num_tokens = attn_metadata.all_rank_num_tokens + inputs_embeds = self.enorm(embed_tokens(input_ids)) + hidden_states = self.hnorm(hidden_states) + hidden_states = torch.cat([inputs_embeds, hidden_states], dim=-1) + mapping = self.model_config.mapping + if mapping.tp_size > 1 and not mapping.enable_attention_dp: + # Row-parallel eh_proj: each rank consumes its own input chunk. + hidden_states = torch.chunk(hidden_states, mapping.tp_size, dim=-1)[mapping.tp_rank] + hidden_states = self.eh_proj(hidden_states) + + runtime_ctx = build_glm5_next_runtime_context(attn_metadata, kv_lens_source="metadata") + residual = hidden_states + attn_in = self.input_layernorm(hidden_states) + parts = [] + if runtime_ctx.num_contexts > 0: + parts.append( + self.self_attn.forward_prefill( + attn_in[: runtime_ctx.num_ctx_tokens], + **runtime_ctx.sparse_kwargs(self.layer_idx, "prefill"), + ) + ) + if runtime_ctx.num_generations > 0: + phase = runtime_ctx.gen_phase + parts.append( + getattr(self.self_attn, f"forward_{phase}")( + attn_in[runtime_ctx.num_ctx_tokens :], + **runtime_ctx.sparse_kwargs(self.layer_idx, phase), + ) + ) + attn_out = parts[0] if len(parts) == 1 else torch.cat(parts, dim=0) + hidden_states = residual + attn_out + + residual = hidden_states + hidden_states = self.mlp(self.post_attention_layernorm(hidden_states), all_rank_num_tokens) + hidden_states = residual + hidden_states + + hidden_states = self.shared_head.norm(hidden_states) + if spec_metadata is not None: + # Two-model-path hook kept for parity with the DeepSeek-V3 MTP layer. + spec_metadata.maybe_capture_hidden_states(0, hidden_states, None) + return hidden_states + + +# --------------------------------------------------------------------------- +# Whole-model materialization +# --------------------------------------------------------------------------- + +#: Routed-expert destinations: consumed by the fused MoE layer's loader rather +#: than placed by name. +_FUSED_MOE_RE = re.compile( + r"^(?Pmodel\.layers\.\d+\.mlp)\.experts\.(?P\d+)" + r"\.(?Pgate_proj|up_proj|down_proj)$" +) diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py new file mode 100644 index 000000000000..2f373f25606d --- /dev/null +++ b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py @@ -0,0 +1,1261 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +"""GLM-5.3-Flash vision tower and multimodal wrapper (``glm5_next``). + +The checkpoint publishes ``Glm5NextForConditionalGeneration``: the accepted +text decoder (:class:`~.modeling_glm5_next.Glm5NextForCausalLM`, hybrid KDA + +sparse-MLA, FP8) plus a 24-block BF16 vision tower. This module adds the +vision side directly on TensorRT-LLM's existing ``Attention`` abstraction — +TRTLLM backend, ``kv_cache_manager=None``, module-side per-head Q/K RMSNorm +and two-axis (h, w) rotary embedding — and composes a thin +``MultimodalModelMixin`` wrapper around the unchanged text decoder. There is +no pure-PyTorch/SDPA production attention path. + +Source semantics (HF ``Glm5NextVisionModel``, the named reference checkout): + +* patch embed: ``Conv3d(3, 1024, kernel=stride=(2, 14, 14))`` over packed + ``(N, 1176)`` patch rows (``3*2*14*14``), emitted by the checkpoint's + ``Glm5NextImageProcessor`` in spatial-merge-block order; +* 24 pre-norm blocks: RMSNorm → biased fused QKV (16 heads × 64) → per-head + RMSNorm on Q and K (fp32 math, eps ``rms_norm_eps``) → 2-axis rotary over + the full head dim in fp32 (16 freqs per axis, theta 10000, neox pairing) + → full non-causal attention within each image segment → biased output + projection; RMSNorm → biased clamped-SwiGLU MLP (limit ``swiglu_limit``); +* post RMSNorm → 2×2 ``Conv2d`` downsample (1024 → 4096) over each + spatial-merge block → projector ``linear → LayerNorm → GELU → clamped + SwiGLU → linear`` (all projector linears bias-free), one 4096-wide row per + image token. + +Per-image full attention reuses the Qwen-VL vision metadata contract +(:func:`~.modeling_qwen2vl._prepare_qwen_vl_vision_attn_metadata`): every +image is one context segment with ``PredefinedAttentionMask.FULL`` and no KV +cache. TP4 shards QKV / gate&up column-wise and o_proj / down row-wise; +norms, convolutions and the projector's first linear stay replicated. + +The prompt contract matches the sealed native-HF MMMU reference: OpenAI-style +content parts with every ```` placeholder interleaved at its original +position (``interleave_placeholders=True``, ``ContentFormat.OPENAI``); the +chat template itself expands each image part to +``<|begin_of_image|><|image|><|end_of_image|>`` and the input processor +expands ``<|image|>`` to ``grid.prod() / spatial_merge_size**2`` image tokens. + +Image and video preprocessing come from the Hugging Face ``Glm5NextProcessor`` +(``AutoProcessor``), exactly as Qwen2-VL and the other VLMs in this tree do; +this uses the GLM-specific Transformers revision documented in the deployment guide. + +Images and videos share the tower. The Hugging Face processor supplies the +timestamped frame structure for videos. +""" + +import copy +import re +from typing import TYPE_CHECKING, Any, Dict, List, Mapping, Optional, Sequence, Tuple + +import numpy as np +import torch +import torch.nn as nn +from PIL import Image +from transformers import AutoProcessor, AutoTokenizer, PretrainedConfig, PreTrainedModel + +from tensorrt_llm._torch.models.modeling_multimodal_utils import _is_mm_disagg +from tensorrt_llm._utils import prefer_pinned + +from ...inputs import ( + ContentFormat, + ExtraProcessedInputs, + MultimodalPlaceholderMetadata, + TextPrompt, + register_input_processor, +) +from ...inputs.multimodal import MultimodalParams +from ...inputs.registry import BaseMultimodalDummyInputsBuilder, BaseMultimodalInputProcessor +from ...logger import logger +from ...sampling_params import SamplingParams +from ..attention.attention import Attention +from ..attention.backends import AttentionMetadata +from ..attention.backends.interface import PredefinedAttentionMask +from ..attention.backends.trtllm import TrtllmAttention +from ..attention.rotary_embedding import RotaryEmbedding +from ..flashinfer_utils import IS_FLASHINFER_AVAILABLE +from ..modules.gated_mlp import GatedMLP +from ..modules.layer_norm import LayerNorm +from ..modules.linear import Linear, TensorParallelMode, WeightMode, WeightsLoadingConfig +from ..modules.rms_norm import RMSNorm +from ..modules.swiglu import swiglu +from ..utils import torch_compiling + +if IS_FLASHINFER_AVAILABLE: + from ..custom_ops import flashinfer_apply_rope_with_cos_sin_cache_inplace +else: # pragma: no cover - CPU / no-flashinfer environments + flashinfer_apply_rope_with_cos_sin_cache_inplace = None +from .modeling_auto import AutoModelForCausalLM +from .modeling_glm5_next import ( + Glm5NextForCausalLM, + get_glm5_next_text_config, + glm5_next_attention_mapping, +) +from .modeling_multimodal_encoder import MultimodalEncoderMixin +from .modeling_multimodal_mixin import ( + EncoderGroup, + MultimodalModelMixin, + encode_multimodal_by_groups, +) +from .modeling_qwen2vl import _prepare_qwen_vl_vision_attn_metadata +from .modeling_utils import ( + ModelConfig, + QuantConfig, + _load_weights_impl, + filter_weights, + register_auto_model, + register_vision_encoder, +) +from .multimodal_encoder_graph import ( + EncoderGraphKey, + EncoderGraphTensorSpec, + EncoderMetadataProvider, + MultimodalEncoderGraphRunner, +) + +if TYPE_CHECKING: + from ...llmapi.llm_args import MultimodalEncoderCudaGraphConfig + +_VISION_WEIGHT_PREFIX = "model.visual" + + +def _image_encoder_cuda_graph_config( + model_config: ModelConfig[PretrainedConfig], +) -> Optional["MultimodalEncoderCudaGraphConfig"]: + mm_config = getattr(model_config, "multimodal_config", None) + if mm_config is None or mm_config.encoder_cuda_graph is None: + return None + unknown = set(mm_config.encoder_cuda_graph) - {"image"} + if unknown: + raise ValueError( + "glm5_next: unsupported multimodal encoder CUDA graph modalities " + f"{sorted(unknown)}; only 'image' is supported" + ) + return mm_config.encoder_cuda_graph.get("image") + + +def _text_dtype(model_config: ModelConfig[PretrainedConfig]) -> torch.dtype: + text_config = get_glm5_next_text_config(model_config.pretrained_config) + dtype = getattr(text_config, "dtype", None) + if isinstance(dtype, str): + dtype = getattr(torch, dtype) + return dtype or torch.bfloat16 + + +def _require_trtllm_vision_backend(model_config: ModelConfig[PretrainedConfig], where: str) -> None: + """Fail closed on any attention backend other than plain TRTLLM. + + The vision bring-up contract admits exactly one production attention + path: the TRTLLM backend with full-mask per-image segments and no KV + cache. VANILLA/FlashInfer (and the generic sparse-attention wrappers, + which swap in a different kernel/metadata contract) must not be + constructible for the vision tower — a configured non-TRTLLM backend is + an error here, never a fallback. The text decoder keeps its own + independently validated configuration. + """ + backend = getattr(model_config, "attn_backend", None) + if not isinstance(backend, str) or backend.upper() != "TRTLLM": + raise ValueError( + f"{where}: the glm5_next vision tower supports only the TRTLLM " + f"attention backend, got attn_backend={backend!r}. There is no " + "VANILLA/FlashInfer/SDPA vision attention path." + ) + if getattr(model_config, "sparse_attention_config", None) is not None: + raise ValueError( + f"{where}: the glm5_next vision tower runs plain full-mask TRTLLM " + "attention; a sparse_attention_config would swap in a sparse " + "backend wrapper and is not supported on the vision path." + ) + + +def _create_linear_weights(*modules: nn.Module) -> None: + """The tower is not a ``DecoderModelForCausalLM``, so nothing runs + ``create_weights`` for it after construction; materialize any shared + module that deferred its weights (``skip_create_weights_in_init``).""" + for module in modules: + for sub in module.modules(): + if isinstance(sub, Linear) and not getattr(sub, "_weights_created", True): + sub.create_weights() + + +class Glm5NextVisionAttention(Attention): + """GLM vision attention on the standard ``Attention`` module. + + Module-side additions relative to the base flow: per-head Q/K RMSNorm + followed by table-driven two-axis rotary embedding. Core execution stays on the + configured backend (TRTLLM) with ``PredefinedAttentionMask.FULL`` per + image segment and no KV cache. + """ + + def __init__(self, model_config: ModelConfig[PretrainedConfig], layer_idx: int) -> None: + _require_trtllm_vision_backend(model_config, type(self).__name__) + config = model_config.pretrained_config.vision_config + text_config = get_glm5_next_text_config(model_config.pretrained_config) + dtype = _text_dtype(model_config) + super().__init__( + hidden_size=config.hidden_size, + num_attention_heads=config.num_heads, + num_key_value_heads=config.num_heads, + max_position_embeddings=int(text_config.max_position_embeddings), + bias=bool(config.attention_bias), + pos_embd_params=None, + rope_fusion=False, + layer_idx=layer_idx, + dtype=dtype, + config=model_config, + reduce_output=( + not model_config.mapping.enable_attention_dp and model_config.mapping.tp_size > 1 + ), + head_dim=config.hidden_size // config.num_heads, + ) + # Hard proof, not just a name check: `get_attention_backend` falls + # back to TRTLLM on unknown names and wraps known ones for sparse + # configs, so verify the constructed backend object is exactly the + # plain TRTLLM implementation. + if type(self.attn) is not TrtllmAttention: + raise ValueError( + f"{type(self).__name__}: constructed attention backend is " + f"{type(self.attn).__name__}, expected TrtllmAttention. The " + "glm5_next vision tower admits no other backend." + ) + # Per-head Q/K RMSNorm over head_dim, weights replicated across TP. + self.q_norm = RMSNorm(hidden_size=self.head_dim, eps=config.rms_norm_eps, dtype=dtype) + self.k_norm = RMSNorm(hidden_size=self.head_dim, eps=config.rms_norm_eps, dtype=dtype) + # Vision attention runs from the outer VL wrapper, outside the + # compiled LM region. Unregister from the shared attn-layer metadata + # map so compiled LM attention lookups never resolve a vision layer + # (same contract as Qwen2_5_VLVisionAttention). + if self.register_to_config: + model_config.extra_attrs.get("attn_layers", {}).pop(self.layer_idx_str, None) + self.register_to_config = False + + def forward( + self, + hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata, + position_embeddings: Tuple[torch.Tensor, torch.Tensor] = None, + **kwargs, + ) -> torch.Tensor: + qkv = self.qkv_proj(hidden_states) + q, k, v = self.split_qkv(qkv, None, None) + seq_len = q.shape[0] + # Per-head Q/K RMSNorm on the shared module, then table-driven RoPE: + # the two-axis (height, width) layout is carried by per-token cos/sin + # rows, so the FlashInfer cos/sin-cache RoPE op applies it with + # positions = row index (the same path Qwen2.5-VL's tower uses); the + # torch helper is the fallback. + q = self.q_norm(q.reshape(-1, self.head_dim)).reshape(seq_len, -1) + k = self.k_norm(k.reshape(-1, self.head_dim)).reshape(seq_len, -1) + cos, sin = position_embeddings + half = self.head_dim // 2 + cos_half, sin_half = cos[:, :half], sin[:, :half] + if flashinfer_apply_rope_with_cos_sin_cache_inplace is not None and self.head_dim % 64 == 0: + q = q.contiguous() + k = k.contiguous() + cos_sin_cache = torch.cat([cos_half, sin_half], dim=-1).to(torch.float32).contiguous() + positions = torch.arange(seq_len, device=q.device, dtype=torch.int32) + flashinfer_apply_rope_with_cos_sin_cache_inplace( + positions, q, k, self.head_dim, cos_sin_cache, is_neox=True + ) + else: + q = ( + RotaryEmbedding.apply_rotary_pos_emb( + q.view(1, seq_len, -1, self.head_dim), cos_half, sin_half, unsqueeze_dim=1 + ) + .to(q.dtype) + .reshape(seq_len, -1) + ) + k = ( + RotaryEmbedding.apply_rotary_pos_emb( + k.view(1, seq_len, -1, self.head_dim), cos_half, sin_half, unsqueeze_dim=1 + ) + .to(k.dtype) + .reshape(seq_len, -1) + ) + # The TRTLLM backend consumes the packed [q|k|v] projection directly. + qkv = torch.cat([q, k, v], dim=-1) + output = self.forward_impl( + q=qkv, + k=None, + v=None, + attn_metadata=attn_metadata, + attention_mask=PredefinedAttentionMask.FULL, + attention_window_size=None, + attention_mask_data=None, + mrope_config=None, + attention_sinks=None, + ) + return self.o_proj(output, layer_idx=self.layer_idx) + + +class Glm5NextVisionBlock(nn.Module): + """Pre-norm residual block: ``x += attn(norm1(x)); x += mlp(norm2(x))``. + + The first residual add is fused into ``norm2``; the block still returns + the full hidden state so boundary hooks see the source activation. + """ + + def __init__(self, model_config: ModelConfig[PretrainedConfig], layer_idx: int) -> None: + super().__init__() + config = model_config.pretrained_config.vision_config + dtype = _text_dtype(model_config) + self.norm1 = RMSNorm(hidden_size=config.hidden_size, eps=config.rms_norm_eps, dtype=dtype) + self.norm2 = RMSNorm(hidden_size=config.hidden_size, eps=config.rms_norm_eps, dtype=dtype) + self.attn = Glm5NextVisionAttention(model_config, layer_idx) + # Biased clamped-SwiGLU MLP: the shared GatedMLP already runs one fused + # gate|up column-sharded GEMM, the clamped swiglu kernel and a + # row-sharded down projection (source ``Glm5NextVisionMLP``). + self.mlp = GatedMLP( + hidden_size=config.hidden_size, + intermediate_size=config.intermediate_size, + bias=bool(config.attention_bias), + dtype=dtype, + config=model_config, + overridden_tp_size=1 if model_config.mapping.enable_attention_dp else None, + layer_idx=layer_idx, + swiglu_limit=float(config.swiglu_limit), + ) + _create_linear_weights(self.mlp, self.attn) + + def forward( + self, + hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata, + position_embeddings: Tuple[torch.Tensor, torch.Tensor], + ) -> torch.Tensor: + attn_out = self.attn( + self.norm1(hidden_states), + attn_metadata=attn_metadata, + position_embeddings=position_embeddings, + ) + normed, hidden_states = self.norm2(attn_out, hidden_states) + return hidden_states + self.mlp(normed) + + +class Glm5NextVisionPatchEmbed(nn.Module): + """``Conv3d`` patchifier over packed ``(N, 3*t*p*p)`` rows (replicated).""" + + def __init__(self, config: PretrainedConfig, dtype: torch.dtype) -> None: + super().__init__() + self.patch_size = config.patch_size + self.temporal_patch_size = config.temporal_patch_size + self.in_channels = config.in_channels + self.embed_dim = config.hidden_size + kernel = (self.temporal_patch_size, self.patch_size, self.patch_size) + self.kernel = kernel + # Meta-safe parameter creation (no fill-style init; the checkpoint + # always overwrites); forward is F.conv3d — Conv3d-exact. + self.proj = nn.Module() + self.proj.weight = nn.Parameter( + torch.empty(self.embed_dim, self.in_channels, *kernel, dtype=dtype) + ) + self.proj.bias = nn.Parameter(torch.empty(self.embed_dim, dtype=dtype)) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + weight = self.proj.weight + # kernel == stride == the packed patch extent, so the Conv3d is exactly + # one GEMM over the flattened (C, T, P, P) patch rows (the cuDNN + # implicit-GEMM conv is ~10x slower than the cuBLAS GEMM here). + hidden_states = hidden_states.reshape(-1, weight.shape[1:].numel()).to(dtype=weight.dtype) + return nn.functional.linear(hidden_states, weight.view(self.embed_dim, -1), self.proj.bias) + + +class Glm5NextVisionPatchMerger(nn.Module): + """Projector: ``proj → LayerNorm → GELU → clamped SwiGLU → down``. + + All linears are bias-free (source ``Glm5NextVisionPatchMerger``); the + first projection and the LayerNorm stay replicated because the norm needs + the full 4096-wide row, gate/up shard column-wise and down row-wise. + """ + + def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: + super().__init__() + config = model_config.pretrained_config.vision_config + dtype = _text_dtype(model_config) + dim = config.out_hidden_size + context_dim = config.projection_intermediate_size + self.swiglu_limit = float(config.swiglu_limit) + # Each attention-DP rank encodes its own images, so the complete + # merger must be local too (the Qwen-VL ownership convention). + mapping = glm5_next_attention_mapping(model_config.mapping) + # Replicated projection + LayerNorm (the norm needs the full row). + self.proj = Linear( + dim, + dim, + bias=False, + dtype=dtype, + mapping=mapping, + quant_config=None, + allreduce_strategy=model_config.allreduce_strategy, + ) + self.post_projection_norm = LayerNorm(hidden_size=dim, eps=1e-5, dtype=dtype) + self.act1 = nn.GELU() + common = dict( + dtype=dtype, + mapping=mapping, + quant_config=None, + allreduce_strategy=model_config.allreduce_strategy, + ) + # One fused gate|up GEMM + the clamped-swiglu kernel, as in the blocks. + self.gate_up_proj = Linear( + dim, + context_dim * 2, + bias=False, + tensor_parallel_mode=TensorParallelMode.COLUMN, + weights_loading_config=WeightsLoadingConfig( + weight_mode=WeightMode.FUSED_GATE_UP_LINEAR + ), + **common, + ) + self.down_proj = Linear( + context_dim, dim, bias=False, tensor_parallel_mode=TensorParallelMode.ROW, **common + ) + + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: + hidden_states = self.proj(hidden_states) + hidden_states = self.act1(self.post_projection_norm(hidden_states)) + gate_up = self.gate_up_proj(hidden_states) + return self.down_proj(swiglu(gate_up, swiglu_limit=self.swiglu_limit)) + + +class Glm5NextVisionModel(nn.Module, MultimodalEncoderMixin): + """The GLM vision tower: patch embed → 24 blocks → post-norm → + downsample → projector. Context-only, ``kv_cache_manager=None``.""" + + def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: + super().__init__() + _require_trtllm_vision_backend(model_config, type(self).__name__) + self.model_config = model_config + self.config = model_config.pretrained_config.vision_config + dtype = _text_dtype(model_config) + + self.spatial_merge_size = self.config.spatial_merge_size + self.spatial_merge_unit = self.spatial_merge_size**2 + self.head_dim = self.config.hidden_size // self.config.num_heads + + self.patch_embed = Glm5NextVisionPatchEmbed(self.config, dtype) + self.blocks = nn.ModuleList( + [Glm5NextVisionBlock(model_config, layer_idx) for layer_idx in range(self.config.depth)] + ) + self.post_layernorm = RMSNorm( + hidden_size=self.config.hidden_size, eps=self.config.rms_norm_eps, dtype=dtype + ) + # Conv2d(kernel = stride = merge) over one merge tile is a GEMM over the + # tile's flattened (kh, kw, C) row; the weight keeps the checkpoint's + # conv layout and is re-viewed once for the GEMM (see patch embed). + self.downsample = nn.Module() + self._downsample_gemm_weight: Optional[torch.Tensor] = None + self.downsample.weight = nn.Parameter( + torch.empty( + self.config.out_hidden_size, + self.config.hidden_size, + self.spatial_merge_size, + self.spatial_merge_size, + dtype=dtype, + ) + ) + self.downsample.bias = nn.Parameter(torch.empty(self.config.out_hidden_size, dtype=dtype)) + self.merger = Glm5NextVisionPatchMerger(model_config) + + # Source rotary: theta 10000 over half the head dim; the two position + # axes (h, w) each get head_dim//4 frequencies and the pair is + # duplicated over both head halves (neox pairing) — fp32 throughout. + # Deliberately NOT a module buffer: the encoder-wide dtype cast would + # downcast it to bf16 (the source keeps its non-persistent inv_freq + # table in fp32), and a 0.4% frequency error at grid positions ~50 + # is a ~1e-2 cos error that compounds across the 24 blocks. + freq_dim = self.head_dim // 2 + # device='cpu' pins the table out of any meta-device construction + # context (it is data, not a weight to materialize later). + self._rope_inv_freq_cpu = 1.0 / ( + 10000.0 ** (torch.arange(0, freq_dim, 2, dtype=torch.float32, device="cpu") / freq_dim) + ) + self._rope_inv_freq_by_device: Dict[torch.device, torch.Tensor] = {} + # Per-grid (cos, sin) tables on the model device; image grids repeat + # heavily in practice and the numpy position build + H2D copy per call + # is pure host latency otherwise. + self._rope_cache: Dict[ + Tuple[Tuple[int, int, int], ...], Tuple[torch.Tensor, torch.Tensor] + ] = {} + self._rope_cache_limit = 64 + + # Pinned, not selected: the backend lock above already rejected every + # non-TRTLLM configuration, so the metadata class is the TRTLLM one by + # construction (no `get_attention_backend` name lookup on this path). + self.metadata_cls = TrtllmAttention.Metadata + self.attn_metadata: Optional[AttentionMetadata] = None + self._fixed_max_seq_len = model_config.max_num_tokens + + # Optional CUDA-graph replay of the block stack (see + # `enable_blocks_cuda_graph`); opted in through + # `multimodal_config.encoder_cuda_graph["image"]`. + self._encoder_cuda_graph_config = _image_encoder_cuda_graph_config(model_config) + self._blocks_graph_runner: Optional[MultimodalEncoderGraphRunner] = None + + @property + def device(self) -> torch.device: + return self.patch_embed.proj.weight.device + + # -- block-stack CUDA graphs --------------------------------------------- + def enable_blocks_cuda_graph( + self, + config: Optional["MultimodalEncoderCudaGraphConfig"] = None, + *, + device: Optional[torch.device] = None, + ) -> None: + """Capture the 24-block loop for every configured bucket. + + Weights must already live on the CUDA device. The eager prelude + (patch embed, rotary tables) and tail (post norm, downsample, merger) + stay outside the graph; a request whose (num_images, total_patches) + matches no bucket falls back to the eager block loop. + """ + config = config if config is not None else self._encoder_cuda_graph_config + if config is None or self._blocks_graph_runner is not None: + return + dtype = self.patch_embed.proj.weight.dtype + runner = MultimodalEncoderGraphRunner( + encoder_fn=self._encoder_graph_fn, + metadata_provider=_Glm5NextVisionGraphMetadataProvider(self), + input_specs={ + "x": EncoderGraphTensorSpec(shape=(self.config.hidden_size,), dtype=dtype), + "cos": EncoderGraphTensorSpec(shape=(self.head_dim,), dtype=torch.float32), + "sin": EncoderGraphTensorSpec(shape=(self.head_dim,), dtype=torch.float32), + }, + output_specs={"x": 0}, + config=config, + ) + runner.capture_all(device if device is not None else self.device) + self._blocks_graph_runner = runner + + def _encoder_graph_fn( + self, inputs: Mapping[str, torch.Tensor], attn_metadata: AttentionMetadata + ) -> Dict[str, torch.Tensor]: + position_embeddings = (inputs["cos"], inputs["sin"]) + return {"x": self._run_blocks_eager(inputs["x"], position_embeddings, attn_metadata)} + + def _run_blocks_eager( + self, + hidden_states: torch.Tensor, + position_embeddings: Tuple[torch.Tensor, torch.Tensor], + attn_metadata: AttentionMetadata, + ) -> torch.Tensor: + for block in self.blocks: + hidden_states = block( + hidden_states, + attn_metadata=attn_metadata, + position_embeddings=position_embeddings, + ) + return hidden_states + + def _run_blocks( + self, + hidden_states: torch.Tensor, + position_embeddings: Tuple[torch.Tensor, torch.Tensor], + seq_lens: List[int], + ) -> torch.Tensor: + # The tower runs outside the compiled LM region with explicit + # metadata; keep both graph replay and eager off the custom-op path. + with torch_compiling(False): + if self._blocks_graph_runner is not None: + cos, sin = position_embeddings + out = self._blocks_graph_runner.maybe_run( + seq_lengths=seq_lens, + inputs={"x": hidden_states, "cos": cos, "sin": sin}, + ) + if out is not None: + return out["x"] + self.attn_metadata = _prepare_qwen_vl_vision_attn_metadata( + seq_lens, self.attn_metadata, max_seq_len=self._fixed_max_seq_len + ) + return self._run_blocks_eager(hidden_states, position_embeddings, self.attn_metadata) + + # -- engine setup contract (MultimodalEncoderMixin) -------------------- + def setup_attn_metadata( + self, + max_num_tokens: int, + attention_metadata_capacity: Optional[Dict[str, int]] = None, + ) -> None: + capacities = ( + attention_metadata_capacity + if attention_metadata_capacity is not None + else self.get_encoder_attention_metadata_capacity(max_num_tokens) + ) + self.attn_metadata = self.metadata_cls( + max_num_requests=capacities["attention"], + max_num_tokens=max_num_tokens, + kv_cache_manager=None, + ) + self.set_attn_max_seq_len(max_num_tokens) + + def set_attn_max_seq_len(self, max_seq_len: int) -> None: + if max_seq_len <= 0: + raise ValueError( + f"GLM vision attention max_seq_len must be positive, got {max_seq_len}" + ) + self._fixed_max_seq_len = max_seq_len + + def get_encoder_attention_metadata_capacity(self, max_num_tokens: int) -> Dict[str, int]: + # Every image segment holds at least one spatial-merge block. + return {"attention": max(1, max_num_tokens // self.spatial_merge_unit)} + + # -- rotary ------------------------------------------------------------- + @staticmethod + def rot_pos_ids(t: int, h: int, w: int, spatial_merge_size: int) -> torch.Tensor: + """(h, w) coordinates per patch in spatial-merge-block order, repeated + over the temporal axis — source ``get_vision_position_ids``.""" + hpos = np.broadcast_to(np.arange(h).reshape(h, 1), (h, w)) + wpos = np.broadcast_to(np.arange(w).reshape(1, w), (h, w)) + block = ( + h // spatial_merge_size, + spatial_merge_size, + w // spatial_merge_size, + spatial_merge_size, + ) + hpos = hpos.reshape(block).transpose(0, 2, 1, 3).flatten() + wpos = wpos.reshape(block).transpose(0, 2, 1, 3).flatten() + pos = torch.from_numpy(np.stack([hpos, wpos], axis=-1).copy()) + if t > 1: + pos = pos.repeat(t, 1) + return pos + + def rot_pos_emb(self, grid_rows: List[List[int]]) -> Tuple[torch.Tensor, torch.Tensor]: + key = tuple(tuple(int(v) for v in row) for row in grid_rows) + cached = self._rope_cache.get(key) + if cached is not None and cached[0].device == self.device: + return cached + pos = torch.cat( + [self.rot_pos_ids(t, h, w, self.spatial_merge_size) for t, h, w in grid_rows], dim=0 + ).to(self.device) + inv_freq = self._rope_inv_freq_by_device.get(pos.device) + if inv_freq is None: + inv_freq = self._rope_inv_freq_cpu.to(pos.device) + self._rope_inv_freq_by_device[pos.device] = inv_freq + freqs = (pos.unsqueeze(-1).float() * inv_freq).flatten(1) + emb = torch.cat((freqs, freqs), dim=-1) + result = (emb.cos(), emb.sin()) + if len(self._rope_cache) >= self._rope_cache_limit: + self._rope_cache.pop(next(iter(self._rope_cache))) + self._rope_cache[key] = result + return result + + # -- forward ------------------------------------------------------------- + @torch.inference_mode() + def forward(self, pixel_values: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor: + grid_rows = [[int(t), int(h), int(w)] for t, h, w in grid_thw.tolist()] + seq_lens: List[int] = [] + for t, h, w in grid_rows: + seq_lens.extend([h * w] * t) + + position_embeddings = self.rot_pos_emb(grid_rows) + hidden_states = self.patch_embed(pixel_values.to(device=self.device)) + hidden_states = self._run_blocks(hidden_states, position_embeddings, seq_lens) + return self.project_merged(hidden_states) + + def project_merged(self, hidden_states: torch.Tensor) -> torch.Tensor: + """Post-blocks tail: post-norm → 2×2 downsample → projector.""" + hidden_states = self.post_layernorm(hidden_states) + merge = self.spatial_merge_size + rows = hidden_states.reshape(-1, merge * merge * hidden_states.shape[-1]) + weight = self.downsample.weight + cached = self._downsample_gemm_weight + if cached is None or cached.device != weight.device or weight.is_meta: + # [out, C, kh, kw] -> [out, kh, kw, C] to match the tile row order. + cached = weight.permute(0, 2, 3, 1).reshape(weight.shape[0], -1).contiguous() + if not weight.is_meta: + self._downsample_gemm_weight = cached + hidden_states = nn.functional.linear(rows, cached, self.downsample.bias) + return self.merger(hidden_states) + + +class _Glm5NextVisionGraphMetadataProvider(EncoderMetadataProvider): + """Graph-owned TRTLLM metadata for the block-stack runner. + + Per bucket one metadata instance with ``is_cuda_graph=True`` so the + ``seq_lens`` setter copies into the captured ``_seq_lens_cuda`` buffer + instead of reallocating; the packed cumulative lengths the FMHA reads + live in a fixed ``cu_q_seqlens`` buffer refreshed in place. Host-side + views are rebound per refresh — the C++ op reads them only at capture. + """ + + graph_critical_attrs: Sequence[str] = ("_seq_lens_cuda", "cu_q_seqlens") + + def __init__(self, vit: "Glm5NextVisionModel") -> None: + self._vit = vit + + def build(self, key: EncoderGraphKey) -> AttentionMetadata: + vit = self._vit + metadata = vit.metadata_cls( + max_num_requests=key.num_contexts, + max_num_tokens=key.total_tokens, + kv_cache_manager=None, + ) + metadata.is_cuda_graph = True + metadata.cu_q_seqlens = torch.zeros( + key.num_contexts + 1, dtype=torch.int32, device=vit.device + ) + metadata.cu_kv_seqlens = metadata.cu_q_seqlens + return metadata + + def refresh_in_place( + self, metadata: AttentionMetadata, padded_seq_lengths: Sequence[int] + ) -> None: + n = len(padded_seq_lengths) + seq_lens = torch.tensor(padded_seq_lengths, dtype=torch.int32, pin_memory=prefer_pinned()) + metadata.num_contexts = n + metadata.request_ids = list(range(1, n + 1)) + metadata.seq_lens = seq_lens + metadata.bind_encoder_cuda_graph_seq_lens(metadata.seq_lens, n) + cu_seqlens = torch.zeros(n + 1, dtype=torch.int32, pin_memory=prefer_pinned()) + torch.cumsum(seq_lens, dim=0, out=cu_seqlens[1:]) + metadata.cu_q_seqlens.copy_(cu_seqlens, non_blocking=True) + metadata.max_seq_len = self._vit._fixed_max_seq_len + metadata.prepare_encoder_only() + + +class Glm5NextVisionModelBase(nn.Module): + """Encoder wrapper: dtype/quant isolation, weight routing, batching.""" + + def __init__( + self, + model_config: ModelConfig[PretrainedConfig], + vlm_base_model: Optional[type[nn.Module]] = None, + ) -> None: + super().__init__() + self.model_config = model_config + self.model_dtype = _text_dtype(model_config) + # The tower is excluded from checkpoint quantization (BF16 in + # ``modules_to_not_convert``); scrub the quant config so no Linear + # picks up FP8 behavior. + self.model_config.quant_config = QuantConfig() + # Every TP collective this model constructs is pinned to NCCL: the + # text decoder pins all of its reductions (AUTO's runtime-selected + # tactic raced at decode on TP4), and the tower's row-parallel + # projections must stay on the same deterministic contract so image + # requests cannot reintroduce the unpinned path. The engine hands the + # wrapper a frozen ModelConfig (and the wrapper hands this class its + # own deepcopy), so use the documented `_frozen` bypass and restore + # the incoming state. + from ..distributed import AllReduceStrategy + + was_frozen = self.model_config._frozen + self.model_config._frozen = False + self.model_config.allreduce_strategy = AllReduceStrategy.NCCL + self.model_config._frozen = was_frozen + self.visual = MultimodalModelMixin._cast_multimodal_encoder_dtype( + (vlm_base_model or Glm5NextVisionModel)(self.model_config), self.model_dtype + ) + + def load_weights( + self, + weights: Dict[str, torch.Tensor], + allow_partial_loading: bool = False, + ) -> None: + """Load ``model.visual.*`` through the shared loader (Qwen2-VL pattern). + + The checkpoint stores one fused ``attn.qkv`` projection and names the + output projection ``attn.proj``; the shared ``Attention`` takes split + q/k/v inputs and ``o_proj``. ``_load_weights_impl`` fuses the block + and merger ``gate_proj``/``up_proj`` pairs into ``gate_up_proj`` and + raises on any parameter left without a source. + """ + visual_weights = filter_weights(_VISION_WEIGHT_PREFIX, weights) + converted: Dict[str, torch.Tensor] = {} + qkv_pattern = re.compile(r"(.*?)attn\.qkv\.(.*)") + for name, tensor in visual_weights.items(): + match = qkv_pattern.match(name) + if match: + prefix, suffix = match.groups() + q, k, v = tensor[:].chunk(3, dim=0) + converted[f"{prefix}attn.q_proj.{suffix}"] = q + converted[f"{prefix}attn.k_proj.{suffix}"] = k + converted[f"{prefix}attn.v_proj.{suffix}"] = v + else: + converted[name] = tensor[:] + self.visual.config.num_attention_heads = self.visual.config.num_heads + _load_weights_impl( + self.visual, + converted, + params_map={r"(.*?)attn\.proj\.(.*)": r"\1attn.o_proj.\2"}, + allow_partial_loading=allow_partial_loading, + ) + + def enable_blocks_cuda_graph(self, *, device: Optional[torch.device] = None) -> None: + """Capture the configured encoder CUDA graphs (no-op when not configured).""" + self.visual.enable_blocks_cuda_graph(device=device) + + @torch.inference_mode() + def encode_batched(self, pixel_values: torch.Tensor, grid_thw: torch.Tensor) -> torch.Tensor: + """Run the tower over HF-processor pixel rows (images and video frame + pairs share one layout: ``[patches, C * T * P * P]``).""" + pixel_values = pixel_values.to(device=self.visual.device, dtype=self.model_dtype) + return self.visual(pixel_values, grid_thw=grid_thw) + + @property + def mm_encoder_groups(self) -> Tuple[EncoderGroup, ...]: + # Images and videos share the tower; the framework lays the output out + # as all image rows then all video rows and reorders into prompt order. + return ( + EncoderGroup( + modalities=("image", "video"), + encoder_fn=self.encode_batched, + build_batched_input=_glm5_next_build_batched_input, + ), + ) + + def forward(self, multimodal_params: List[MultimodalParams]) -> List[torch.Tensor]: + return [encode_multimodal_by_groups(self.mm_encoder_groups, multimodal_params)] + + +def _flatten_video_grid_thw(video_grid_thw: torch.Tensor) -> torch.Tensor: + """Source ``Glm5NextModel.get_video_features``: every temporal frame pair of + a video is encoded as its own ``(1, h, w)`` item, so ``(t, h, w)`` becomes + ``t`` rows of ``(1, h, w)``.""" + t = video_grid_thw[:, 0] + hw = torch.repeat_interleave(video_grid_thw[:, 1:], t, dim=0) + return torch.cat([hw.new_ones(hw.shape[0], 1), hw], dim=1) + + +def _glm5_next_build_batched_input(multimodal_params: List[MultimodalParams]) -> Dict[str, Any]: + pixels: List[torch.Tensor] = [] + grids: List[torch.Tensor] = [] + for param in multimodal_params: + bucket = param.multimodal_data.get("image") + if bucket is not None: + pixels.append(bucket["pixel_values"]) + grids.append(bucket["image_grid_thw"]) + for param in multimodal_params: + bucket = param.multimodal_data.get("video") + if bucket is not None: + pixels.append(bucket["pixel_values_videos"]) + grids.append(_flatten_video_grid_thw(bucket["video_grid_thw"])) + device = pixels[0].device + return { + "pixel_values": torch.cat(pixels, dim=0), + "grid_thw": torch.cat([g.to(device) for g in grids], dim=0), + } + + +class Glm5NextInputProcessor(BaseMultimodalInputProcessor, BaseMultimodalDummyInputsBuilder): + """Input processor on the Hugging Face ``Glm5NextProcessor`` (image and + video processors + placeholder expansion), the Qwen2-VL pattern. + + Uses the GLM-specific Transformers revision documented in the deployment guide. The + processor performs smart resize, + normalization, patchification, frame sampling / timestamped video token + layout and the ``<|image|>`` / ``<|video|>`` expansion; this class only + routes the outputs into the engine's payload format. + """ + + def __init__( + self, + model_path: str, + config: PretrainedConfig, + tokenizer: Optional[AutoTokenizer] = None, + trust_remote_code: bool = True, + **kwargs, + ): + super().__init__( + model_path=model_path, + config=config, + tokenizer=tokenizer, + trust_remote_code=trust_remote_code, + **kwargs, + ) + text_config = get_glm5_next_text_config(config) + dtype = getattr(text_config, "dtype", None) or torch.bfloat16 + self._dtype = getattr(torch, dtype) if isinstance(dtype, str) else dtype + self._tokenizer = ( + tokenizer if tokenizer is not None else AutoTokenizer.from_pretrained(model_path) + ) + self._processor = AutoProcessor.from_pretrained( + model_path, use_fast=self._use_fast, trust_remote_code=trust_remote_code + ) + vision = config.vision_config + self._merge_size = int(vision.spatial_merge_size) + self._patch_size = int(vision.patch_size) + + @property + def config(self) -> PretrainedConfig: + return self._config + + @property + def tokenizer(self) -> AutoTokenizer: + return self._tokenizer + + @property + def model_path(self) -> str: + return self._model_path + + @property + def processor(self) -> AutoProcessor: + return self._processor + + @property + def dtype(self) -> torch.dtype: + return self._dtype + + def get_vocab_size(self) -> int: + return int(get_glm5_next_text_config(self._config).vocab_size) + + def get_preferred_media_io_kwargs(self) -> Dict[str, Dict[str, Any]]: + # PIL is the HF processor's native input; the server's default float + # CHW tensor would be decoded, hashed and converted back for nothing. + return {"image": {"format": "pil"}} + + def get_mm_token_ids(self) -> torch.Tensor: + """In-vocab ``<|image|>`` / ``<|video|>`` ids: the frontend's + out-of-vocabulary fallback (``ids >= vocab_size``) would find no + multimodal positions and the engine would skip fusion (Qwen2-VL + precedent). The begin/end delimiters stay ordinary text.""" + ids = [ + int(tid) + for tid in ( + getattr(self._config, "image_token_id", None), + getattr(self._config, "video_token_id", None), + ) + if tid is not None + ] + return torch.tensor(ids, dtype=torch.int32) + + @property + def spatial_merge_unit(self) -> int: + return self._merge_size**2 + + def get_num_tokens_per_video( + self, + *, + video: List[Any], + video_metadata: Optional[dict] = None, + video_grid_thw: Optional[torch.Tensor] = None, + **kwargs, + ) -> int: + """Embedding rows one video contributes: ``video_grid_thw.prod() / + merge_size**2`` after the HF video processor's temporal pairing. + + The framework passes the processor-produced ``video_grid_thw`` when it + has it (Qwen3-VL precedent); otherwise the frames are run through the + HF video processor. (The processor's own + ``_get_num_multimodal_tokens(video_sizes=...)`` counts frames before + pairing and would overstate the video by ``temporal_patch_size``.) + """ + if video_grid_thw is None: + meta = dict(video_metadata or {}) + meta["total_num_frames"] = len(video) + video_grid_thw = self._processor.video_processor( + videos=[video], video_metadata=[meta], do_sample_frames=False, return_tensors="pt" + )["video_grid_thw"] + grid = torch.as_tensor(video_grid_thw) + return int(grid.prod(dim=-1).sum().item()) // self.spatial_merge_unit + + # -- profiling dummies ---------------------------------------------------- + def _max_grid_side(self, max_patches: int) -> int: + side = int(max_patches**0.5) + side -= side % self._merge_size + while side > 0 and side * side > max_patches: + side -= self._merge_size + return max(side, self._merge_size) + + def _processor_max_patches(self) -> int: + """The HF image processor's own patch cap (its ``max_pixels`` budget), + measured by asking it about an oversized square image.""" + big = 64 * 1024 + return int( + self._processor._get_num_multimodal_tokens(image_sizes=[(big, big)])[ + "num_image_patches" + ][0] + ) + + def get_mm_max_tokens_per_item( + self, max_num_encoder_tokens: Optional[int] = None + ) -> Dict[str, int]: + max_patches = self._processor_max_patches() + if max_num_encoder_tokens is not None: + max_patches = min(max_patches, int(max_num_encoder_tokens)) + side = self._max_grid_side(max_patches) + return {"image": side * side} + + def get_dummy_mm_data( + self, + *, + max_num_encoder_tokens: int, + mm_counts: Mapping[str, int], + dtype: Optional[torch.dtype] = None, + ) -> Dict[str, Any]: + num_images = int(mm_counts.get("image", 0)) + if num_images <= 0: + return {} + per_item = max(self.spatial_merge_unit, int(max_num_encoder_tokens) // num_images) + side = self._max_grid_side(min(per_item, self._processor_max_patches())) + pixels_side = side * self._patch_size + image = Image.new("RGB", (pixels_side, pixels_side), (127, 127, 127)) + out = self._processor.image_processor(images=[image] * num_images, return_tensors="pt") + return { + "image": { + "pixel_values": out["pixel_values"].to(dtype or self._dtype), + "image_grid_thw": out["image_grid_thw"], + } + } + + # -- request processing ---------------------------------------------------- + def call_with_text_prompt( + self, inputs: TextPrompt, sampling_params: SamplingParams + ) -> Tuple[List[int], Optional[ExtraProcessedInputs]]: + text_prompt = inputs.get("prompt") + mm_data = inputs.get("multi_modal_data") or {} + mm_processor_kwargs = dict(inputs.get("mm_processor_kwargs") or {}) + images = mm_data.get("image") or None + video_datas = mm_data.get("video") or None + if not images and not video_datas: + # Text-only fast path: identical to the tokenizer and no + # multimodal payload, so the tower is never touched. + input_ids = self._tokenizer(text_prompt, return_tensors="pt").input_ids + return input_ids[0].to(torch.int32).tolist(), None + + # Media loaders may deliver pre-rescaled float tensors (the server + # default) instead of PIL / uint8; tell the HF processor not to + # rescale those a second time (Qwen2-VL precedent). + do_rescale = True + if images and isinstance(images[0], torch.Tensor): + do_rescale = False + videos = None + video_metadata = None + if video_datas: + videos = [video_data.frames for video_data in video_datas] + if isinstance(videos[0][0], torch.Tensor): + do_rescale = False + # Frames are already sampled by the media loader; the processor + # only needs fps / frame indices to lay out the per-frame + # timestamps (``VideoMetadata.timestamps``). + video_metadata = [] + for video_data in video_datas: + meta = dict(video_data.metadata or {}) + meta["total_num_frames"] = len(video_data.frames) + video_metadata.append(meta) + mm_processor_kwargs.setdefault("do_sample_frames", False) + + # Fail closed before any pixel work: a placeholder / item count + # mismatch must surface as a ValueError (an HTTP error at the serving + # boundary), not as the HF processor's iterator exhaustion. + for placeholder, items, what in ( + (self._processor.image_token, images or [], "image"), + (self._processor.video_token, video_datas or [], "video"), + ): + n_placeholders = text_prompt.count(placeholder) + if n_placeholders != len(items): + raise ValueError( + f"glm5_next multimodal request carries {n_placeholders} " + f"{placeholder!r} placeholder(s) but {len(items)} {what}(s); " + "refusing the mismatched request" + ) + + processed = self._processor( + text=[text_prompt], + images=images, + videos=videos, + video_metadata=video_metadata, + do_rescale=do_rescale, + return_tensors="pt", + **mm_processor_kwargs, + ) + multimodal_data: Dict[str, Any] = {} + if processed.get("pixel_values") is not None: + multimodal_data["image"] = { + "pixel_values": processed["pixel_values"].to(self._dtype), + "image_grid_thw": processed["image_grid_thw"], + } + if processed.get("pixel_values_videos") is not None: + multimodal_data["video"] = { + "pixel_values_videos": processed["pixel_values_videos"].to(self._dtype), + "video_grid_thw": processed["video_grid_thw"], + } + fused_input_ids = processed["input_ids"][0] + return fused_input_ids.to(torch.int32).tolist(), {"multimodal_data": multimodal_data} + + +@register_vision_encoder(Glm5NextVisionModelBase, vlm_base_model=Glm5NextVisionModel) +@register_auto_model("Glm5NextForConditionalGeneration") +@register_input_processor( + Glm5NextInputProcessor, + model_type="glm5_next", + placeholder_metadata=MultimodalPlaceholderMetadata( + # The chat template consumes OpenAI-style content parts natively and + # emits `<|begin_of_image|><|image|><|end_of_image|>` per image part; + # interleaving preserves each image's original position in the text — + # the contract the sealed native-HF MMMU reference recorded. + placeholder_map={ + "image": "<|begin_of_image|><|image|><|end_of_image|>", + "video": "<|begin_of_video|><|video|><|end_of_video|>", + }, + placeholders_separator="", + content_format=ContentFormat.OPENAI, + interleave_placeholders=True, + ), +) +class Glm5NextVLM(MultimodalModelMixin, PreTrainedModel): + """Thin conditional-generation wrapper around the accepted text decoder. + + The inner LM is the unchanged :class:`Glm5NextForCausalLM` (constructed + through its own registered architecture name), so text-only requests run + the sealed text path byte-for-byte — the vision processor/tower is only + reachable when a request carries image or video data. The text decoder keeps + ``KVCacheManagerV2`` ownership; the vision tower runs context-only with + no cache manager. + """ + + _supports_flash_attn = True + _supports_sdpa = True + + def _check_and_adjust_experts_implementation(self, *args, **kwargs): + # Transformers 5.x PreTrainedModel.__init__ probes MoE expert + # implementations; the wrapper holds no MoE modules itself (the text + # decoder manages its own experts), so skip the check. + return None + + def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs) -> None: + config = model_config.pretrained_config + self.original_arch = config.architectures[0] + super().__init__(config) + self.model_config = model_config + + llm_model_config = copy.deepcopy(model_config) + # Share the live extra_attrs dict: the LM attention layers register + # their per-layer metadata there and model_engine reads the same dict. + llm_model_config.extra_attrs = model_config.extra_attrs + llm_model_config.pretrained_config.architectures = ["Glm5NextForCausalLM"] + self.llm = AutoModelForCausalLM.from_config(llm_model_config) + + self.mm_encoder = None + if not (_is_mm_disagg() or model_config.disable_mm_encoder): + self.mm_encoder = Glm5NextVisionModelBase(copy.deepcopy(model_config)).eval() + self.mm_encoder_groups = self.mm_encoder.mm_encoder_groups + else: + logger.info("Glm5NextVLM: multimodal encoder disabled; serving text-only requests.") + + # device='cpu' keeps the constant real under meta-device construction. + self._mm_token_ids = torch.tensor( + [int(config.image_token_id), int(config.video_token_id)], + dtype=torch.int32, + device="cpu", + ) + self.post_config() + + # -- engine contracts ----------------------------------------------------- + @classmethod + def get_preferred_kv_cache_manager_version(cls, pretrained_config=None) -> str: + return Glm5NextForCausalLM.get_preferred_kv_cache_manager_version(pretrained_config) + + @property + def mamba_metadata_cls(self): + # The engine resolves the Mamba metadata class from the top-level model. + return self.llm.mamba_metadata_cls + + @property + def multimodal_token_ids(self) -> torch.Tensor: + return self._mm_token_ids + + @property + def multimodal_data_device_paths(self) -> List[str]: + return ["image.pixel_values", "video.pixel_values_videos", "multimodal_embedding"] + + @property + def language_model(self) -> torch.nn.Module: + return self.llm + + # Speculative decoding lives on the inner decoder (``SpecDecOneEngineForCausalLM`` + # via the deep-copied model_config), but ``ModelLoader.load`` reads the draft + # state from the outer model it resolved for the checkpoint's architecture. + # ``load_draft_weights`` keeps an explicit signature: the loader dispatches + # kwargs via ``inspect.getfullargspec``. + @property + def draft_config(self): + return self.llm.draft_config + + @property + def draft_model(self): + return self.llm.draft_model + + def load_draft_weights( + self, weights: Dict[str, torch.Tensor], weight_mapper: Optional[Any] = None + ): + return self.llm.load_draft_weights(weights, weight_mapper=weight_mapper) + + def get_language_model_extra_forward_kwargs( + self, + *, + raw_input_ids: Optional[torch.Tensor], + position_ids: Optional[torch.Tensor], + mm_inputs: Any, + spec_metadata: Any = None, + resource_manager: Any = None, + **forward_kwargs: Any, + ) -> Dict[str, Any]: + # The mixin's default forwards only the five common arguments. The + # decoder's speculative worker indexes spec_metadata unconditionally, + # and MTP drafting needs the pre-fusion token ids once fused + # inputs_embeds replace input_ids for image prompts (Qwen3-VL contract). + del position_ids, mm_inputs, forward_kwargs + return { + "spec_metadata": spec_metadata, + "resource_manager": resource_manager, + "orig_input_ids": raw_input_ids, + } + + @property + def text_embedding_layer(self): + return self.llm.model.embed_tokens + + @property + def embedding_dim(self) -> int: + return self.text_embedding_layer.embedding_dim + + @property + def embedding_dtype(self) -> torch.dtype: + return self.text_embedding_layer.weight.dtype + + def post_config(self): + self.model_config.pretrained_config = self.llm.config + self.config = self.model_config.pretrained_config + + def encode_multimodal_inputs( + self, multimodal_params: List[MultimodalParams], **encoder_kwargs: Any + ) -> torch.Tensor: + if self.mm_encoder is None: + raise ValueError("Raw multimodal inputs require a local multimodal encoder.") + mm_embeds = self.mm_encoder.forward(list(multimodal_params), **encoder_kwargs) + if len(mm_embeds) != 1: + raise ValueError( + "glm5_next multimodal encoder must return one packed " + f"embedding tensor, but returned {len(mm_embeds)} tensors." + ) + return mm_embeds[0] + + def load_weights(self, weights: Dict[str, torch.Tensor], **_ignored: Any) -> None: + """Route ``model.visual.*`` to the tower, everything else to the + audited text loader (which itself allowlists the visual namespace). + + The wrapper does not declare ``weight_mapper`` because the text + decoder initializes its own registered mapper. An explicitly passed + wrapper-level mapper is rejected. + """ + if _ignored.pop("weight_mapper", None) is not None: + raise ValueError( + "glm5_next uses its audited exact-placement loader; a " + "checkpoint-format weight_mapper is not supported" + ) + if self.mm_encoder is not None: + visual_keys = any(key.startswith(_VISION_WEIGHT_PREFIX) for key in weights) + if not visual_keys: + raise ValueError( + "glm5_next multimodal load: the checkpoint holds no " + f"'{_VISION_WEIGHT_PREFIX}.*' weights but the vision " + "encoder is enabled; refusing to leave an uninitialized " + "tower (serve text-only with disable_mm_encoder instead)" + ) + self.mm_encoder.load_weights(weights) + # Weights are on device now; capture the (opt-in) encoder graphs. + self.mm_encoder.enable_blocks_cuda_graph() + self.llm.load_weights(weights) diff --git a/tensorrt_llm/_torch/models/modeling_speculative.py b/tensorrt_llm/_torch/models/modeling_speculative.py index b1978d20fa93..8f450d020f14 100644 --- a/tensorrt_llm/_torch/models/modeling_speculative.py +++ b/tensorrt_llm/_torch/models/modeling_speculative.py @@ -1322,6 +1322,9 @@ def __init__( case "deepseek_v4": from .modeling_deepseekv4 import DeepseekV4MTP mtp_layer = DeepseekV4MTP + case "glm5_next" | "glm5_next_text": + from .modeling_glm5_next import Glm5NextMTP + mtp_layer = Glm5NextMTP case _: raise ValueError( f"Model type {model_type} not supported for MTP") diff --git a/tensorrt_llm/_torch/modules/kimi_kda/_kda_decode.py b/tensorrt_llm/_torch/modules/kimi_kda/_kda_decode.py index 34feb497faa0..9fee4c0cc913 100644 --- a/tensorrt_llm/_torch/modules/kimi_kda/_kda_decode.py +++ b/tensorrt_llm/_torch/modules/kimi_kda/_kda_decode.py @@ -126,9 +126,9 @@ def run_kda_decode_fusion_cuda( HV = x_v.shape[2] if x_k.shape[1:3] != (B, H) or x_v.shape[1] != B: raise ValueError("x_q, x_k, and x_v batch/head dimensions are inconsistent") - if H != HV or H not in (1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 96): + if H != HV or H not in (1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64, 96): raise ValueError( - "CUDA KDA decode fusion supports H == HV in {1,2,3,4,6,8,12,16,24,32,48,96}" + "CUDA KDA decode fusion supports H == HV in {1,2,3,4,6,8,12,16,24,32,48,64,96}" ) if ssm_state_indices is None and not state.is_contiguous(): raise ValueError("state must be contiguous because it is updated in place") diff --git a/tensorrt_llm/_torch/modules/kimi_kda/kimi_kda_mixer.py b/tensorrt_llm/_torch/modules/kimi_kda/kimi_kda_mixer.py index 794952787dd7..99857c0f17c8 100644 --- a/tensorrt_llm/_torch/modules/kimi_kda/kimi_kda_mixer.py +++ b/tensorrt_llm/_torch/modules/kimi_kda/kimi_kda_mixer.py @@ -274,8 +274,13 @@ def projection(name: str, in_features: int, out_features: int) -> nn.Module: # Fused prefill/decode/verify projection weights, built after checkpoint # load. BF16 uses separate fused [q | k | v | g] and [f_a | b] # GEMMs; FP8 fuses QKVG from checkpoint codes/scales and keeps BFA BF16. + # Low-rank BF16 gates instead use [q | k | v], [f_a | g_a | b], + # and batched [f_b; g_b]. FP8 fusion remains full-rank only. self._qkvg_proj_weight: Optional[torch.Tensor] = None self._bfa_proj_weight: Optional[torch.Tensor] = None + # Low-rank gate only: ``[2, head_dim, D]`` transposed view of the + # stacked ``[f_b; g_b]`` weights for one batched GEMM. + self._gate_b_t: Optional[torch.Tensor] = None self._w_q_t = self._w_k_t = self._w_v_t = None self._A_log_f32 = self._dt_bias_f32 = self._onorm_w_f32 = None # Fork/join state for overlapping the small [f_a | b] -> f_b chain @@ -312,23 +317,37 @@ def finalize_decode_weights(self) -> None: transposed conv weights (bf16 ``[W, D]``) and fp32 copies of ``A_log`` / ``dt_bias`` / ``o_norm.weight``. """ - if self._dispatch.decode_kernel_path != "optimized" or not self.use_full_rank_gate: + if self._dispatch.decode_kernel_path != "optimized": return if self.q_proj.weight.device.type != "cuda": return with torch.no_grad(): - qkvg_modules = (self.q_proj, self.k_proj, self.v_proj, self.g_proj) + if self.use_full_rank_gate: + qkvg_modules = ( + self.q_proj, + self.k_proj, + self.v_proj, + self.g_proj, + ) + bfa_modules = (self.f_a_proj, self.b_proj) + else: + # Low-rank gate: the output gate's ``g_a`` rows ride with the + # other head-independent projections reading the same input + # (``[f_a | g_a | b]``), and ``[f_b; g_b]`` become one batched + # GEMM over adjacent columns of that result. + qkvg_modules = (self.q_proj, self.k_proj, self.v_proj) + bfa_modules = (self.f_a_proj, self.g_a_proj, self.b_proj) + gate_b = self._merge_projection_weights((self.f_b_proj, self.g_b_proj)) + self._gate_b_t = gate_b.view(2, self.proj_size, self.head_dim).transpose(1, 2) if all(isinstance(module, nn.Linear) for module in qkvg_modules): self._qkvg_proj_weight = self._merge_projection_weights(qkvg_modules) - elif all( + elif self.use_full_rank_gate and all( isinstance(module, Linear) and module.has_fp8_block_scales for module in qkvg_modules ): self.qkvg_proj = self._fuse_checkpoint_projections(qkvg_modules) self.qkvg_split_sizes = [module.out_features for module in qkvg_modules] - self._bfa_proj_weight = self._merge_projection_weights( - (self.f_a_proj, self.b_proj), pad_rows_to=8 - ) + self._bfa_proj_weight = self._merge_projection_weights(bfa_modules, pad_rows_to=8) self._build_decode_kernel_constants() @staticmethod @@ -524,6 +543,28 @@ def _sync_kda_replay_conv_window(self, layer_cache, slot_indices, conv_pool) -> return layer_cache.commit_conv_window(slot_indices, conv_pool) + def _project_gate_inputs( + self, x: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor]]: + """``(beta, forget_gate, onorm_g)`` from the fused ``[f_a | (g_a) | b]`` GEMM. + + ``onorm_g`` is the low-rank output gate ``g_b(g_a(x))`` (from the + batched ``[f_b; g_b]`` GEMM) or ``None`` for the full-rank gate, whose + output gate rides in the fused qkvg projection instead. Requires + ``finalize_decode_weights`` to have published ``_bfa_proj_weight``. + """ + hd, H = self.head_dim, self.num_heads + bfa = torch.nn.functional.linear(x, self._bfa_proj_weight) + if self.use_full_rank_gate: + return bfa[..., hd : hd + H], self.f_b_proj(bfa[..., :hd]), None + beta = bfa[..., 2 * hd : 2 * hd + H] + lead = bfa.shape[:-1] + # [N, 2, hd] -> [2, N, hd] strided view of the adjacent f_a / g_a + # columns; cuBLAS takes the strided batch directly (no stack copy). + pair = bfa[..., : 2 * hd].reshape(-1, 2, hd).transpose(0, 1) + gates = torch.bmm(pair, self._gate_b_t) # [2, N, D] + return beta, gates[0].reshape(*lead, -1), gates[1].reshape(*lead, -1) + def _project_packed_conv_input( self, x: torch.Tensor, x2d: torch.Tensor ) -> Tuple[torch.Tensor, Optional[torch.Tensor]]: @@ -537,7 +578,11 @@ def _project_packed_conv_input( # Transposing the GEMM skips the repack the paths below still need. weight = self._qkvg_proj_weight packed_conv = torch.mm(weight[: 3 * d], x2d.t()) - onorm_g = torch.nn.functional.linear(x, weight[3 * d : 4 * d]) + onorm_g = ( + torch.nn.functional.linear(x, weight[3 * d : 4 * d]) + if self.use_full_rank_gate + else None + ) return packed_conv, onorm_g onorm_g = None @@ -639,10 +684,10 @@ def forward_prefill( ) if self._bfa_proj_weight is not None: - bfa = torch.nn.functional.linear(x, self._bfa_proj_weight) - f_a = bfa[..., : self.head_dim] - beta = bfa[..., self.head_dim : self.head_dim + self.num_heads].float() - g = self.f_b_proj(f_a) + beta, g, onorm_lowrank = self._project_gate_inputs(x) + beta = beta.float() + if onorm_g is None: + onorm_g = onorm_lowrank else: g = self.f_b_proj(self.f_a_proj(x)) beta = self.b_proj(x).float() @@ -799,24 +844,19 @@ def _project_qkvg() -> torch.Tensor: return torch.nn.functional.linear(x2d, self._qkvg_proj_weight) return self.qkvg_proj(x2d) - def _project_bfa_and_fb() -> tuple[torch.Tensor, torch.Tensor]: - bfa = torch.nn.functional.linear(x2d, self._bfa_proj_weight) - f_a = bfa[:, :hd] - beta = bfa[:, hd : hd + H] - return beta, self.f_b_proj(f_a) - projection_aux_stream = ( self._projection_aux_stream if B <= _KDA_BFA_MULTISTREAM_MAX_ROWS else None ) - qkvg, (beta, g) = maybe_execute_in_parallel( + qkvg, (beta, g, onorm_lowrank) = maybe_execute_in_parallel( _project_qkvg, - _project_bfa_and_fb, + lambda: self._project_gate_inputs(x2d), self._projection_fork_event, self._projection_join_event, projection_aux_stream, disable_on_compile=True, ) - x_qkvg = qkvg[:, : 4 * d] + x_qkvg = qkvg[:, : 3 * d] + onorm_g = qkvg[:, 3 * d : 4 * d] if self.use_full_rank_gate else onorm_lowrank # Section views retain the live pool's slot stride, including V2 # manager padding. The kernel uses ssm_state_indices for both pools. @@ -842,7 +882,7 @@ def _project_bfa_and_fb() -> tuple[torch.Tensor, torch.Tensor]: dt_bias=self._dt_bias_f32, beta=beta.unsqueeze(0), state=kernel_state, - onorm_g=x_qkvg[:, 3 * d :].unflatten(-1, (H, hd)).unsqueeze(0), + onorm_g=onorm_g.unflatten(-1, (H, hd)).unsqueeze(0), onorm_weight=self._onorm_w_f32, out=kda_out, ssm_state_indices=kernel_state_indices, @@ -1077,23 +1117,16 @@ def _project_qkvg() -> torch.Tensor: return torch.nn.functional.linear(x, qkvg_weight) return fused_qkvg(x) - bfa_weight = self._bfa_proj_weight - if bfa_weight is not None: - - def _project_bfa_and_fb() -> tuple[torch.Tensor, torch.Tensor]: - bfa = torch.nn.functional.linear(x, bfa_weight) - f_a = bfa[..., : self.head_dim] - beta = bfa[..., self.head_dim : self.head_dim + self.num_heads] - return beta, self.f_b_proj(f_a) - + onorm_lowrank = None + if self._bfa_proj_weight is not None: projection_aux_stream = ( self._projection_aux_stream if 0 < num_rows <= _KDA_BFA_MULTISTREAM_MAX_ROWS else None ) - qkvg, (beta, forget_gate) = maybe_execute_in_parallel( + qkvg, (beta, forget_gate, onorm_lowrank) = maybe_execute_in_parallel( _project_qkvg, - _project_bfa_and_fb, + lambda: self._project_gate_inputs(x), self._projection_fork_event, self._projection_join_event, projection_aux_stream, @@ -1107,10 +1140,15 @@ def _project_bfa_and_fb() -> tuple[torch.Tensor, torch.Tensor]: d = self.proj_size q_proj, k_proj, v_proj = (part.contiguous() for part in qkvg[..., : 3 * d].split(d, dim=-1)) qkvg_split_sizes = self.qkvg_split_sizes - has_onorm_gate = qkvg_weight is not None or ( - self.use_full_rank_gate and qkvg_split_sizes is not None and len(qkvg_split_sizes) == 4 + has_onorm_gate = self.use_full_rank_gate and ( + qkvg_weight is not None or (qkvg_split_sizes is not None and len(qkvg_split_sizes) == 4) ) - onorm_g = qkvg[..., 3 * d : 4 * d].contiguous() if has_onorm_gate else None + if has_onorm_gate: + onorm_g = qkvg[..., 3 * d : 4 * d].contiguous() + elif onorm_lowrank is not None: + onorm_g = onorm_lowrank.contiguous() + else: + onorm_g = None return q_proj, k_proj, v_proj, forget_gate, beta, onorm_g def forward_verify_fused( diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index 660ffe11b601..35ad04c9e5b4 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -52,6 +52,7 @@ get_num_spec_layers, get_spec_decoder, should_use_separate_draft_kv_cache) from ..utils import is_gdn_replay_enabled +from . import config_utils from .config_utils import (MambaKVCacheParams, _is_sliding_attention_layer, extract_mamba_kv_cache_params, extract_qwen4_exp_ple_cache_params, @@ -239,6 +240,43 @@ def get_kv_cache_manager_cls( "use_kv_cache_manager_v2=True; V1 supports only " "periodic_snapshot_interval.") + if config_utils.is_glm5_next(config): + # GLM-5.3-Flash: one Glm5NextCacheManager (a + # MambaHybridCacheManagerV2 subclass) owns the KDA recurrent/conv + # states, the sparse-MLA latent pages, and the per-sparse-layer + # INDEX_KEY indexer buffers. The indexer state is a V2 extra + # buffer, so no V1/Mixed/Cpp manager can express it: conflicting + # knobs fail loudly instead of silently selecting a manager that + # would drop the indexer cache. + if use_py_mamba_cache_manager() or os.environ.get( + 'TLLM_MAMBA_MANAGER_PREFERENCE'): + raise ValueError( + "glm5_next supports only its V2 cache manager; unset " + "TRTLLM_USE_PY_MAMBA / TLLM_MAMBA_MANAGER_PREFERENCE.") + if is_disagg: + # Only the Python NIXL transceiver moves the KDA recurrent + # state and the V2 extra (indexer) buffers; the C++ + # transceiver would silently drop both. Same rule as the + # hybrid V2 branch below, checked here so the message names + # the model. + backend, runtime = _resolve_disagg_transceiver_route( + cache_transceiver_config) + if runtime != "PYTHON" or backend != "NIXL": + raise ValueError( + "glm5_next disaggregated serving requires " + "cache_transceiver_config backend='NIXL' with " + "transceiver_runtime='PYTHON' (got " + f"backend={backend!r}, transceiver_runtime={runtime!r})." + ) + if not use_v2: + raise ValueError( + "glm5_next requires KV cache manager V2 (the sparse-layer " + "indexer state is a V2 extra buffer). Leave " + "kv_cache_config.use_kv_cache_manager_v2='auto' or set it " + "to True.") + from ..models.modeling_glm5_next import glm5_next_cache_manager_cls + return glm5_next_cache_manager_cls() + # Kimi K3 (KDA + MLA hybrid): block reuse uses the unified C++ pool # (CppMambaHybridCacheManager) like the other hybrid linear models — # per-block KDA state snapshots every mamba_state_cache_interval @@ -2963,7 +3001,96 @@ def _create_kv_cache_manager( if issubclass(kv_cache_manager_cls, MambaHybridCacheManagerV2): manager_extra_kwargs["is_disagg"] = is_disagg - if is_kimi_linear(config): + if config_utils.is_glm5_next(config): + # GLM-5.3-Flash hybrid: KDA recurrent/conv states on the mamba side, + # sparse-MLA latent cache (num_kv_heads=1, head_dim = kv_lora_rank + + # qk_rope_head_dim, SELFKONLY) plus one Role.INDEX_KEY indexer buffer + # per sparse layer on the paged side, all owned by one + # Glm5NextCacheManager. Must come before the is_mla(...) route: the + # glm5_next text config carries MLA fields, but only 11 of its 45 + # layers are sparse MLA. + if max_beam_width > 1: + raise ValueError("glm5_next + beam search is not supported yet.") + if not estimating_kv_cache and kv_connector_manager is not None: + raise NotImplementedError( + "Connector manager is not supported for glm5_next.") + text_config = config_utils.unwrap_glm5_next_text_config(config) + mamba_params = extract_mamba_kv_cache_params( + config, + spec_config=spec_config, + quant_config=quant_config, + ) + mamba_layer_mask, full_attention_layer_mask = ( + _get_mamba_cache_layer_masks( + mamba_params, + mapping, + spec_config, + is_draft, + )) + num_mamba_layers = (0 if is_draft and mamba_params.num_draft_layers > 0 + else mamba_params.num_mamba_layers) + # The indexer state rides the same layer ids as the sparse latent + # pages; both are slot-addressed through the manager's slot-major + # views (see Glm5NextCacheManager.get_batch_slot_tables). + sparse_layer_ids = [ + i for i, is_sparse in enumerate(full_attention_layer_mask) + if is_sparse + ] + # KDA fused multi-token verify (trtllm::kda_mtp_decode, shared with + # Kimi K3): the per-slot replay caches replace the per-step + # intermediate verification buffers. glm5_next has no other verify + # path, so the kernel must be available when speculating. + glm_extra_kwargs = {} + if spec_config is not None: + from ..modules.kimi_kda._kda_kernels import \ + is_kda_mtp_verify_available + if not is_kda_mtp_verify_available(): + raise RuntimeError( + "glm5_next speculative decoding requires the fused KDA " + "verify kernel (trtllm::kda_mtp_decode, CUTLASS DSL), " + "which is unavailable on this device/build.") + glm_extra_kwargs["kda_replay_num_spec"] = ( + spec_config.tokens_per_gen_step - 1) + kv_cache_manager = kv_cache_manager_cls( + # mamba (KDA) cache parameters + mamba_params.state_size, + mamba_params.conv_kernel, + mamba_params.num_heads, + mamba_params.n_groups, + mamba_params.head_dim, + num_mamba_layers, + mamba_layer_mask, + mamba_params.dtype, + mamba_params.mamba_ssm_cache_dtype, + # kv cache parameters (sparse-MLA latent + indexer state) + kv_cache_config, + tensorrt_llm.bindings.internal.batch_manager.CacheType.SELFKONLY, + num_layers=sum(full_attention_layer_mask), + layer_mask=full_attention_layer_mask, + num_kv_heads=1, + head_dim=int(text_config.kv_lora_rank) + + int(getattr(text_config, "qk_rope_head_dim", 0) or 0), + tokens_per_block=tokens_per_block, + max_seq_len=max_seq_len, + max_num_tokens=max_num_tokens, + is_draft=is_draft, + max_batch_size=max_batch_size, + mapping=mapping, + dtype=kv_cache_dtype, + spec_config=spec_config, + is_estimating_kv_cache=estimating_kv_cache, + execution_stream=execution_stream, + sparse_layer_ids=sparse_layer_ids, + # One indexer cache row is [k | gate | pool key] (see + # Glm5NextIndexer.cache_state_dim). + index_state_dim=3 * int(text_config.index_head_dim), + # KDA's conv state is a [Q | K | V] concatenation whose three + # sections have identical width, i.e. the qwen3_next layout. + **_mamba_conv_layout_kwargs(kv_cache_manager_cls, "qwen3_next"), + **glm_extra_kwargs, + **manager_extra_kwargs, + ) + elif is_kimi_linear(config): # Kimi K3 hybrid: KDA (Kimi Delta Attention) recurrent/conv states on # the mamba side of the hybrid manager, absorbed-MQA MLA latent cache # (num_kv_heads=1, head_dim = kv_lora_rank + qk_rope_head_dim, diff --git a/tensorrt_llm/_torch/pyexecutor/config_utils.py b/tensorrt_llm/_torch/pyexecutor/config_utils.py index 268cf0f08c07..f8b9fdc83324 100644 --- a/tensorrt_llm/_torch/pyexecutor/config_utils.py +++ b/tensorrt_llm/_torch/pyexecutor/config_utils.py @@ -137,7 +137,7 @@ def is_gemma4_hybrid(config): def is_hybrid_linear(config): return is_nemotron_hybrid(config) or is_qwen3_hybrid(config) or \ - is_kimi_linear(config) or is_qwen4_exp(config) + is_kimi_linear(config) or is_qwen4_exp(config) or is_glm5_next(config) def is_kimi_linear(config): @@ -200,6 +200,71 @@ def get_kimi_linear_num_attention_layers(config): return sum(full_mask) +def is_glm5_next(config: transformers.PretrainedConfig) -> bool: + """True for GLM-5.3-Flash ("glm5_next") hybrid KDA + sparse-MLA text models. + + Handles both the flattened text config (model_type "glm5_next_text") and + the composite VLM config (model_type "glm5_next" with a nested + text_config). + """ + model_type = getattr(config, "model_type", None) + if model_type == "glm5_next_text": + return getattr(config, "layer_types", None) is not None + if model_type == "glm5_next": + text_config = getattr(config, "text_config", None) + return text_config is not None and is_glm5_next(text_config) + return False + + +def unwrap_glm5_next_text_config( + config: transformers.PretrainedConfig) -> transformers.PretrainedConfig: + """Return the flattened GLM-5.3-Flash text config. + + ``is_glm5_next`` accepts both the flattened text config and the composite + "glm5_next" config with a nested ``text_config``; consumers read + text-level fields (``layer_types``, ``linear_attn_config``, + ``kv_lora_rank``, ...), so they must unwrap the composite form first. + """ + if getattr(config, "model_type", None) == "glm5_next": + text_config = getattr(config, "text_config", None) + if text_config is not None: + return text_config + return config + + +def get_glm5_next_layer_masks( + config: transformers.PretrainedConfig) -> tuple[list[bool], list[bool]]: + """Return (full_attention_layer_mask, kda_layer_mask) for GLM-5.3-Flash. + + The text config's literal ``layer_types`` list names every decoder layer + as exactly one of ``linear_attention`` (KDA) or + ``deepseek_sparse_attention`` (sparse MLA + indexer); 34 + 11 on the real + checkpoint. + """ + config = unwrap_glm5_next_text_config(config) + if len(config.layer_types) != config.num_hidden_layers: + raise ValueError( + "glm5_next layer_types must contain num_hidden_layers entries") + full_mask, kda_mask = [], [] + for layer_idx, layer_type in enumerate(config.layer_types): + is_kda = layer_type == "linear_attention" + is_full = layer_type == "deepseek_sparse_attention" + if is_kda == is_full: + raise ValueError( + f"glm5_next layer {layer_idx} must be exactly one of " + f"linear_attention / deepseek_sparse_attention; got " + f"{layer_type!r}") + kda_mask.append(is_kda) + full_mask.append(is_full) + return full_mask, kda_mask + + +def get_glm5_next_num_attention_layers( + config: transformers.PretrainedConfig) -> int: + full_mask, _ = get_glm5_next_layer_masks(config) + return sum(full_mask) + + def _coerce_torch_dtype(dtype): """Normalize dtype values from HF configs into torch dtype objects. @@ -672,6 +737,18 @@ def extract_mamba_kv_cache_params( n_groups = lin["num_heads"] head_dim = lin["head_dim"] target_full_attn_mask, mamba_mask = get_kimi_linear_layer_masks(config) + elif is_glm5_next(config): + # GLM-5.3-Flash KDA state uses the same mamba parametrization as + # Kimi K3: [q | k | v] short-conv sections of identical width and an + # [H, V, K] fp32 delta-rule recurrent state, with the KDA geometry + # coming from the text config's linear_attn_config. + lin = unwrap_glm5_next_text_config(config).linear_attn_config + state_size = lin["head_dim"] + conv_kernel = lin["short_conv_kernel_size"] + num_heads = lin["num_heads"] + n_groups = lin["num_heads"] + head_dim = lin["head_dim"] + target_full_attn_mask, mamba_mask = get_glm5_next_layer_masks(config) else: raise ValueError( f"{type(config).__name__} is not a supported hybrid Mamba config") @@ -691,6 +768,10 @@ def extract_mamba_kv_cache_params( mamba_ssm_cache_dtype = resolve_auto_ssm_cache_dtype( config, torch.bfloat16) validate_kimi_kda_state_dtype(config, mamba_ssm_cache_dtype) + if is_glm5_next(config) and mamba_ssm_cache_dtype != torch.float32: + logger.info(f"GLM KDA: overriding mamba_ssm_cache_dtype " + f"{mamba_ssm_cache_dtype} -> torch.float32") + mamba_ssm_cache_dtype = torch.float32 return MambaKVCacheParams( state_size=state_size, diff --git a/tensorrt_llm/usage/architecture_allowlist.py b/tensorrt_llm/usage/architecture_allowlist.py index d8295d349883..84a42c44f614 100644 --- a/tensorrt_llm/usage/architecture_allowlist.py +++ b/tensorrt_llm/usage/architecture_allowlist.py @@ -46,6 +46,7 @@ "Gemma4UnifiedForConditionalGeneration", "Glm4MoeForCausalLM", "Glm4MoeLiteForCausalLM", + "Glm5NextForConditionalGeneration", "GlmMoeDsaForCausalLM", "GptOssForCausalLM", "GraniteForCausalLM", diff --git a/tests/integration/defs/accuracy/references/acceptance_length.yaml b/tests/integration/defs/accuracy/references/acceptance_length.yaml index 7a5fce89a575..d99893474a34 100644 --- a/tests/integration/defs/accuracy/references/acceptance_length.yaml +++ b/tests/integration/defs/accuracy/references/acceptance_length.yaml @@ -116,3 +116,9 @@ ModelingV2DeepseekR10528Nvfp4Sm103Dep4::mtp3: # and a subtly wrong one ~2.1, both far below this, while every accuracy # gate stays green. It leaves modeling_v2 17% of headroom and stock 20%. min_al: 2.5 +# GLM-5.3-Flash: the checkpoint's single MTP layer chained for 3 drafts, +# B200 TP4/EP4, GSM8K under the harness protocol (5-shot, no chat template, +# 256 output tokens). min_al is 95% of the measured value. +TestGLM53FlashFP8::test_mtp: + ref_al: 3.384 + min_al: 3.215 diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml index bf3bc2bb1431..747bc9177fe5 100644 --- a/tests/integration/defs/accuracy/references/gsm8k.yaml +++ b/tests/integration/defs/accuracy/references/gsm8k.yaml @@ -542,3 +542,10 @@ zai-org/GLM-5.2: - quant_algo: NVFP4 kv_cache_quant_algo: NVFP4 accuracy: 90 +zai-org/GLM-5.3-Flash: + # B200 TP4/EP4, FP8 block-scale checkpoint, harness protocol (5-shot, no chat template). + - quant_algo: FP8_BLOCK_SCALES + accuracy: 92.57 + - quant_algo: FP8_BLOCK_SCALES + spec_dec_algo: MTP + accuracy: 92.91 diff --git a/tests/integration/defs/accuracy/references/mmmu.yaml b/tests/integration/defs/accuracy/references/mmmu.yaml index 9579749348b1..36f1a0aeaa99 100644 --- a/tests/integration/defs/accuracy/references/mmmu.yaml +++ b/tests/integration/defs/accuracy/references/mmmu.yaml @@ -72,3 +72,10 @@ moonshotai/Kimi-K2.5: # quant_algo=None. moonshotai/Kimi-K3: - accuracy: 84.89 +# GLM-5.3-Flash (glm5_next VLM: 24-block BF16 vision tower + KDA/sparse-MLA +# hybrid FP8 decoder). B200 TP4/EP4, thinking model scored through +# strip_thinking_and_extract_mmmu_answer with a 4096-token output budget and +# reasoning_effort=max; single-run reference (lm-eval stderr +/-1.34 at n=900). +zai-org/GLM-5.3-Flash: + - quant_algo: FP8_BLOCK_SCALES + accuracy: 77.22 diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index d6f8287c998e..175a2df2da80 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -2235,6 +2235,98 @@ def test_fp8_nixl_python(self, mocker, snapshot_policy): extra_evaluator_kwargs={GSM8K: self.GSM8K_EVALUATOR_KWARGS}) +@pytest.mark.timeout(DEFAULT_TEST_TIMEOUT) +@skip_pre_blackwell +@pytest.mark.skip_less_device_memory(80000) +class TestGLM53FlashFP8(LlmapiAccuracyTestHarness): + """GLM-5.3-Flash (glm5_next): hybrid KDA + sparse-MLA layers. + + Only the Python NIXL transceiver moves the recurrent state and the indexer + side cache between workers, so the C++ transceiver is not an option here. + """ + MODEL_NAME = "zai-org/GLM-5.3-Flash" + MODEL_PATH = f"{llm_models_root()}/GLM-5.3-Flash" + + @pytest.mark.skip_less_device(8) + @parametrize_with_ids("mtp", [False, True]) + def test_fp8_nixl(self, mtp): + # Import the selected agent, including the automatic Python fallback + # when C++ bindings are unavailable. A Python-package-only check + # would incorrectly skip builds that use the C++ agent. + pytest.importorskip( + "tensorrt_llm._torch.disaggregation.nixl.agent", + reason="selected NIXL transfer agent is unavailable", + exc_type=ImportError, + ) + kv_cache_config = { + "free_gpu_memory_fraction": 0.5, + "enable_block_reuse": False, + } + cache_transceiver_config = { + "backend": "NIXL", + "transceiver_runtime": "PYTHON", + } + # The checkpoint's single MTP layer chained for three drafts; both + # workers carry it so the context worker hands over a replay-ready + # KDA state and the generation worker verifies on the fused kernel. + speculative_config = { + "decoding_type": "MTP", + "max_draft_len": 3, + } if mtp else None + ctx_server_config = { + "tensor_parallel_size": 4, + "pipeline_parallel_size": 1, + "moe_expert_parallel_size": 4, + "max_batch_size": 64, + "max_num_tokens": 16384, + "max_seq_len": 8192, + "disable_overlap_scheduler": True, + "cuda_graph_config": None, + "kv_cache_config": kv_cache_config, + "speculative_config": speculative_config, + "cache_transceiver_config": cache_transceiver_config, + } + gen_server_config = { + "tensor_parallel_size": 4, + "pipeline_parallel_size": 1, + "moe_expert_parallel_size": 4, + "max_batch_size": 64, + "max_num_tokens": 16384, + "max_seq_len": 8192, + "disable_overlap_scheduler": False, + "cuda_graph_config": { + "max_batch_size": 64, + "enable_padding": True, + }, + "kv_cache_config": kv_cache_config, + "speculative_config": speculative_config, + "cache_transceiver_config": cache_transceiver_config, + } + disaggregated_server_config = { + "hostname": "localhost", + "backend": "pytorch", + "context_servers": { + "num_instances": 1 + }, + "generation_servers": { + "num_instances": 1 + } + } + with launch_disaggregated_llm(disaggregated_server_config, + ctx_server_config, + gen_server_config, + self.MODEL_PATH, + max_workers=64) as llm: + # launch_disaggregated_llm builds a bare LlmArgs for the DuckLLM; + # fill in the quantization so the reference lookup matches the + # registered FP8_BLOCK_SCALES entry. + llm.args.quant_config.quant_algo = "FP8_BLOCK_SCALES" + if mtp: + llm.args.speculative_config = MTPDecodingConfig( + max_draft_len=speculative_config["max_draft_len"]) + run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) + + @pytest.mark.timeout(DEFAULT_TEST_TIMEOUT) @skip_pre_blackwell @pytest.mark.skip_less_device_memory(80000) diff --git a/tests/integration/defs/accuracy/test_glm53_flash.py b/tests/integration/defs/accuracy/test_glm53_flash.py new file mode 100644 index 000000000000..a0937185882e --- /dev/null +++ b/tests/integration/defs/accuracy/test_glm53_flash.py @@ -0,0 +1,426 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import gc + +import pytest +import torch + +from tensorrt_llm import LLM +from tensorrt_llm.evaluate.post_processing import strip_thinking_and_extract_mmmu_answer +from tensorrt_llm.llmapi import ( + CudaGraphConfig, + KvCacheConfig, + MambaStateConfig, + MTPDecodingConfig, + SamplingParams, +) +from tensorrt_llm.quantization import QuantAlgo + +from ..conftest import llm_models_root, parametrize_with_ids, skip_pre_blackwell +from .accuracy_core import ( + GSM8K, + MMMU, + ForceTokenLogitsProcessor, + LlmapiAccuracyTestHarness, + assert_acceptance_length_for_llm, +) + + +class TestGLM53FlashFP8(LlmapiAccuracyTestHarness): + """GLM-5.3-Flash FP8 accuracy and runtime tests on B200. + + Block reuse is enabled only in the periodic-snapshot test. + """ + + MODEL_NAME = "zai-org/GLM-5.3-Flash" + MODEL_PATH = f"{llm_models_root()}/GLM-5.3-Flash" + + @staticmethod + def _llm_kwargs(tp_size: int, ep_size: int) -> dict: + return dict( + tensor_parallel_size=tp_size, + pipeline_parallel_size=1, + moe_expert_parallel_size=ep_size, + kv_cache_config=KvCacheConfig(free_gpu_memory_fraction=0.5, enable_block_reuse=False), + max_batch_size=64, + max_num_tokens=16384, + max_seq_len=8192, + cuda_graph_config=CudaGraphConfig(max_batch_size=64, enable_padding=True), + disable_overlap_scheduler=False, + ) + + @staticmethod + def _assert_glm5_next_stack(llm: LLM) -> None: + assert llm.args.quant_config.quant_algo == QuantAlgo.FP8_BLOCK_SCALES + assert llm.args.kv_cache_config.enable_block_reuse is False + assert llm.args.cuda_graph_config is not None + assert llm.args.cuda_graph_config.enable_padding is True + + @skip_pre_blackwell + @pytest.mark.skip_less_mpi_world_size(4) + @parametrize_with_ids("tp_size,ep_size", [(4, 4)]) + def test_tep(self, tp_size, ep_size): + with LLM(self.MODEL_PATH, **self._llm_kwargs(tp_size, ep_size)) as llm: + self._assert_glm5_next_stack(llm) + assert llm.args.speculative_config is None + task = GSM8K(self.MODEL_NAME) + task.evaluate(llm) + + @skip_pre_blackwell + @pytest.mark.skip_less_mpi_world_size(4) + @parametrize_with_ids("tp_size,ep_size", [(4, 4)]) + def test_attention_dp(self, tp_size, ep_size): + """Attention data parallelism. + + Replicated KDA / sparse-MLA / dense weights per rank, the fused MoE + combining across ranks. + """ + kwargs = self._llm_kwargs(tp_size, ep_size) + kwargs["enable_attention_dp"] = True + # Every rank prefills its own batch and the fused MoE gathers all + # ranks' tokens, so the per-rank token budget is a quarter of the + # tensor-parallel one for the same activation peak. + kwargs["max_num_tokens"] = 4096 + with LLM(self.MODEL_PATH, **kwargs) as llm: + self._assert_glm5_next_stack(llm) + assert llm.args.enable_attention_dp is True + task = GSM8K(self.MODEL_NAME) + task.evaluate(llm) + + @skip_pre_blackwell + @pytest.mark.skip_less_mpi_world_size(4) + @parametrize_with_ids("tp_size,ep_size", [(4, 4)]) + def test_block_reuse(self, tp_size, ep_size): + """KV block reuse with periodic Mamba state snapshots. + + The hybrid V2 manager silently disables reuse unless a snapshot + policy is set; with one, a prefix hit restores the KDA recurrent + state from the snapshot and reuses the latent / indexer pages. The + 5-shot GSM8K prompts share a long prefix, so nearly every request + exercises the path. + """ + kwargs = self._llm_kwargs(tp_size, ep_size) + kwargs["kv_cache_config"] = KvCacheConfig( + free_gpu_memory_fraction=0.5, + enable_block_reuse=True, + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=256), + ) + with LLM(self.MODEL_PATH, return_perf_metrics=True, **kwargs) as llm: + assert llm.args.kv_cache_config.enable_block_reuse is True + assert llm.args.quant_config.quant_algo == QuantAlgo.FP8_BLOCK_SCALES + self._assert_kv_cache_reuse(llm) + task = GSM8K(self.MODEL_NAME) + task.evaluate(llm) + + @staticmethod + def _assert_kv_cache_reuse(llm: LLM) -> None: + # Long enough to cross a 256-token snapshot point; the warm request + # must report reused blocks and reproduce the cold request's tokens. + prompt = llm.tokenizer.encode( + "The capital of France is Paris. The capital of Germany is Berlin. " * 40 + ) + sampling_params = SamplingParams( + max_tokens=8, temperature=0, end_id=-1, return_perf_metrics=True + ) + cold = llm.generate([prompt], sampling_params=sampling_params, use_tqdm=False)[0].outputs[0] + warm = llm.generate([prompt], sampling_params=sampling_params, use_tqdm=False)[0].outputs[0] + assert cold.request_perf_metrics.kv_cache_metrics.num_reused_blocks == 0 + assert warm.request_perf_metrics.kv_cache_metrics.num_reused_blocks > 0 + assert warm.token_ids == cold.token_ids + + # MMMU through the multimodal wrapper (image inputs). The thinking model + # answers inside , so the K2.5 strip-thinking extractor scores it; + # reasoning_effort is the checkpoint's own chat-template knob. Output budget + # follows the native-HF reference protocol (4096 generated tokens). + MMMU_EXTRA_EVALUATOR_KWARGS = dict( + chat_template_kwargs={"reasoning_effort": "max"}, + post_process_fn=strip_thinking_and_extract_mmmu_answer, + preserve_caller_max_tokens=True, + ) + + @skip_pre_blackwell + @pytest.mark.skip_less_mpi_world_size(4) + @pytest.mark.timeout(7200) + @parametrize_with_ids("tp_size,ep_size", [(4, 4)]) + def test_mmmu(self, tp_size, ep_size): + kwargs = self._llm_kwargs(tp_size, ep_size) + # MMMU prompts fit in 8K (MAX_INPUT_LEN); a smaller token budget and + # batch keep the run cheap. Profiling peaks are the same as the text + # tests' (about 92 GiB per GPU at 16K / batch 64 on B200). + kwargs["max_num_tokens"] = MMMU.MAX_INPUT_LEN + kwargs["max_seq_len"] = MMMU.MAX_INPUT_LEN + 4096 + kwargs["max_batch_size"] = 32 + kwargs["cuda_graph_config"] = CudaGraphConfig(max_batch_size=32, enable_padding=True) + with LLM(self.MODEL_PATH, **kwargs) as llm: + self._assert_glm5_next_stack(llm) + task = MMMU(self.MODEL_NAME) + task.evaluate( + llm, + sampling_params=SamplingParams( + max_tokens=4096, truncate_prompt_tokens=MMMU.MAX_INPUT_LEN + ), + extra_evaluator_kwargs=self.MMMU_EXTRA_EVALUATOR_KWARGS, + ) + + @skip_pre_blackwell + @pytest.mark.skip_less_device(4) + @pytest.mark.timeout(1800) + def test_video_url(self, tmp_path): + """Exercise GLM video ingestion through the OpenAI serving endpoint.""" + import json + import sys + from pathlib import Path + + import yaml + from openai import OpenAI + + from ..common import get_free_port_in_ci + from ..examples.serve.test_serve import _wait_for_server_ready + from ..trt_test_alternative import popen + + pytest.importorskip("cv2", reason="video decoding requires OpenCV") + pytest.importorskip("transformers.models.glm5_next.processing_glm5_next") + video = Path(llm_models_root()) / "multimodals/test_data/OAI-sora-tokyo-walk.mp4" + if not video.is_file(): + pytest.skip(f"video fixture is unavailable: {video}") + config_path = tmp_path / "video.yaml" + config_path.write_text( + yaml.safe_dump( + { + "kv_cache_config": { + "free_gpu_memory_fraction": 0.5, + "enable_block_reuse": False, + }, + "cuda_graph_config": {"max_batch_size": 4, "enable_padding": True}, + "enable_chunked_prefill": True, + } + ) + ) + port = get_free_port_in_ci() + command = [ + sys.executable, + "-m", + "tensorrt_llm.commands.serve", + self.MODEL_PATH, + "--host", + "127.0.0.1", + "--port", + str(port), + "--served_model_name", + self.MODEL_NAME, + "--tp_size", + "4", + "--ep_size", + "4", + "--max_batch_size", + "4", + "--max_num_tokens", + "8192", + "--max_seq_len", + "8192", + "--config", + str(config_path), + "--media_io_kwargs", + json.dumps({"video": {"num_frames": 4, "fps": 2}}), + ] + with popen(command) as process: + _wait_for_server_ready(process, http_port=port, timeout=1200) + with OpenAI(base_url=f"http://127.0.0.1:{port}/v1", api_key="tensorrt_llm") as client: + response = client.chat.completions.create( + model=self.MODEL_NAME, + messages=[ + { + "role": "user", + "content": [ + {"type": "video_url", "video_url": {"url": str(video)}}, + { + "type": "text", + "text": "Describe what happens in this video briefly.", + }, + ], + } + ], + temperature=0, + max_completion_tokens=256, + extra_body={"chat_template_kwargs": {"reasoning_effort": "low"}}, + ) + assert len(response.choices) == 1 + message = response.choices[0].message + # Thinking output may fill the short budget before the final answer. + assert (message.content or getattr(message, "reasoning_content", "") or "").strip() + assert response.usage.completion_tokens > 0 + + @skip_pre_blackwell + @pytest.mark.skip_less_mpi_world_size(4) + @parametrize_with_ids("tp_size,ep_size", [(4, 4)]) + def test_mtp(self, tp_size, ep_size): + # The checkpoint's single MTP layer is chained for three drafts per + # step; KDA verification runs on the fused replay kernel. + with LLM( + self.MODEL_PATH, + speculative_config=MTPDecodingConfig(max_draft_len=3), + max_stats_len=-1, + enable_iter_perf_stats=True, + **self._llm_kwargs(tp_size, ep_size), + ) as llm: + self._assert_glm5_next_stack(llm) + assert llm.args.speculative_config.max_draft_len == 3 + task = GSM8K(self.MODEL_NAME) + task.evaluate(llm) + assert_acceptance_length_for_llm("TestGLM53FlashFP8::test_mtp", llm) + + @skip_pre_blackwell + @pytest.mark.timeout(900) + @pytest.mark.skip_less_mpi_world_size(4) + @pytest.mark.threadleak(enabled=False) + def test_runtime(self) -> None: + """One model load, several runtime contracts on short prompts.""" + num_ranks = 4 + max_num_tokens = 256 + with LLM( + self.MODEL_PATH, + tensor_parallel_size=num_ranks, + pipeline_parallel_size=1, + moe_expert_parallel_size=num_ranks, + kv_cache_config=KvCacheConfig(free_gpu_memory_fraction=0.5, enable_block_reuse=False), + max_batch_size=num_ranks, + max_num_tokens=max_num_tokens, + max_seq_len=2048, + enable_chunked_prefill=True, + cuda_graph_config=None, + disable_overlap_scheduler=True, + enable_autotuner=False, + return_perf_metrics=True, + ) as llm: + assert llm.args.enable_chunked_prefill is True + assert llm.args.max_num_tokens == max_num_tokens + self._assert_chunked_prefill(llm) + self._assert_logits_processor(llm) + self._assert_repeated_request_is_deterministic(llm) + + @skip_pre_blackwell + @pytest.mark.skip_less_mpi_world_size(4) + @pytest.mark.timeout(1800) + def test_chunked_prefill_parity(self) -> None: + """Compare the first-token choice and probability across chunk boundaries.""" + logits = [] + prompt_ids = None + for chunked in (False, True): + kwargs = self._llm_kwargs(4, 4) + kwargs.update( + max_batch_size=1, + max_seq_len=2048, + max_num_tokens=256 if chunked else 1024, + enable_chunked_prefill=chunked, + cuda_graph_config=None, + disable_overlap_scheduler=True, + enable_autotuner=False, + return_perf_metrics=True, + disable_mm_encoder=True, + ) + with LLM(self.MODEL_PATH, **kwargs) as llm: + if prompt_ids is None: + text = "The capital of France is Paris. The capital of Germany is Berlin. " + prompt_ids = llm.tokenizer.encode(text * 128)[:768] + assert len(prompt_ids) == 768 + result = llm.generate( + [prompt_ids], + sampling_params=SamplingParams( + max_tokens=1, + temperature=0, + end_id=-1, + return_generation_logits=True, + return_perf_metrics=True, + ), + use_tqdm=False, + )[0] + logits.append(result.outputs[0].generation_logits[0].float().cpu().clone()) + chunks = result.time_breakdown_metrics["ctx_chunk_metrics"] + if chunked: + assert len(chunks) >= 3 + else: + assert len(chunks) == 1 + del result, llm + gc.collect() + torch.cuda.empty_cache() + # Follow the Nemotron MoE BCG/eager comparison: FP8 GEMM changes can + # change routing, so strict whole-logit equality is not a stable gate. + # Require the chunked choice to remain in the reference top-2 and + # bound the reference choice's log-probability change (2.30 nats). + assert torch.isfinite(torch.stack(logits)).all() + reference_top2 = logits[0].topk(2).indices + assert logits[1].argmax() in reference_top2 + logprobs = torch.log_softmax(torch.stack(logits), dim=-1) + difference = (logprobs[0, reference_top2[0]] - logprobs[1, reference_top2[0]]).abs() + assert difference < 2.30, f"first-token log-probability changed by {difference.item()} nats" + + @staticmethod + def _assert_chunked_prefill(llm: LLM) -> None: + prompt_length = 768 + output_length = 16 + prompt_token_ids = [1] + [44] * (prompt_length - 2) + [45] + outputs = llm.generate( + [prompt_token_ids], + sampling_params=SamplingParams(max_tokens=output_length, temperature=0, end_id=-1), + use_tqdm=False, + ) + assert isinstance(outputs, list) + assert len(outputs) == 1 + assert len(outputs[0].outputs[0].token_ids) == output_length + time_breakdown = outputs[0].time_breakdown_metrics + assert time_breakdown is not None + context_chunks = time_breakdown.get("ctx_chunk_metrics") + assert isinstance(context_chunks, list) + # 768 prompt tokens at max_num_tokens=256: three context chunks, each + # continuing the KDA recurrent state and the sparse pools. + assert len(context_chunks) >= 3 + + @staticmethod + def _assert_logits_processor(llm: LLM) -> None: + forced_token_id = 22 + output_length = 4 + outputs = llm.generate( + [[1, 42, 43]], + sampling_params=SamplingParams( + max_tokens=output_length, + temperature=0, + end_id=-1, + logits_processor=ForceTokenLogitsProcessor(forced_token_id), + ), + use_tqdm=False, + ) + assert isinstance(outputs, list) + assert len(outputs) == 1 + assert outputs[0].outputs[0].token_ids == [forced_token_id] * output_length + + @staticmethod + def _assert_repeated_request_is_deterministic(llm: LLM) -> None: + """The same greedy natural-language request decoded twice yields identical tokens. + + Same instance, same batch composition: the hybrid cache must hand a + fresh request a clean KDA / sparse slot, so leftover state from the + previous (chunked, 768-token) occupant would show up as a token flip + here. Natural text keeps the logits peaked; on flat, garbage-token + prompts the decode kernels' run-to-run numeric jitter alone flips + greedy choices, which is not what this checks. + """ + output_length = 8 + prompt = llm.tokenizer.encode("The capital of France is Paris. The capital of Germany is") + sampling_params = SamplingParams(max_tokens=output_length, temperature=0, end_id=-1) + first = llm.generate([prompt], sampling_params=sampling_params, use_tqdm=False)[0] + second = llm.generate([prompt], sampling_params=sampling_params, use_tqdm=False)[0] + assert len(first.outputs[0].token_ids) == output_length + assert second.outputs[0].token_ids == first.outputs[0].token_ids diff --git a/tests/integration/test_lists/test-db/l0_b200.yml b/tests/integration/test_lists/test-db/l0_b200.yml index 189e878f763b..456697cc3676 100644 --- a/tests/integration/test_lists/test-db/l0_b200.yml +++ b/tests/integration/test_lists/test-db/l0_b200.yml @@ -116,6 +116,7 @@ l0_b200: - unittest/_torch/attention/sparse/deepseek_v4/test_compressor_tf32.py TIMEOUT (15) - unittest/_torch/attention/sparse/test_sparse_mla_forward.py TIMEOUT (60) - unittest/_torch/modeling/test_modeling_deepseekv4.py + - unittest/_torch/modeling/test_glm5_next_contracts.py - unittest/llmapi/test_deepseek_v4_tokenizer.py - unittest/_torch/modules/test_mhc.py - unittest/_torch/modules/test_engram.py diff --git a/tests/unittest/_torch/attention/sparse/glm_kpool/__init__.py b/tests/unittest/_torch/attention/sparse/glm_kpool/__init__.py new file mode 100644 index 000000000000..52a7a9daf028 --- /dev/null +++ b/tests/unittest/_torch/attention/sparse/glm_kpool/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 diff --git a/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py b/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py new file mode 100644 index 000000000000..e0f3a0104b1c --- /dev/null +++ b/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py @@ -0,0 +1,151 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""CPU tests for GLM sparse attention metadata and cache layouts.""" + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch + +from tensorrt_llm._torch.attention.backends.sparse.glm_kpool import ( + INDEX_SENTINEL, + GlmKpoolSparseAttention, + GlmKpoolSparseParams, + latent_pool_rows, + paged_slot_indices, + positions_to_pool_rows, +) + +pytestmark = pytest.mark.cpu_only + + +def test_cache_state_prefers_live_metadata_and_rejects_graph_fallback(): + backend = object.__new__(GlmKpoolSparseAttention) + backend.layer_idx = 0 + latent = torch.zeros(4, 8, 1, 512, dtype=torch.bfloat16) + index = torch.zeros(4, 8, 1, 384, dtype=torch.bfloat16) + tables = torch.tensor([[2, 0], [3, 1]], dtype=torch.long) + live_lengths = torch.tensor([5, 8], dtype=torch.int32) + stale_lengths = torch.tensor([9, 12], dtype=torch.long) + metadata = SimpleNamespace( + kv_cache_manager=SimpleNamespace( + tokens_per_block=8, + get_latent_state_buffer=lambda _: latent, + get_index_state_buffer=lambda _: index, + ), + seq_lens=torch.ones(2, dtype=torch.long), + num_contexts=0, + is_cuda_graph=True, + kv_lens_cuda=live_lengths, + mamba_metadata=SimpleNamespace(glm_block_tables=tables, glm_kv_lens=stale_lengths), + ) + state = backend._cache_state(metadata) + assert state.block_tables.data_ptr() == tables.data_ptr() + assert state.kv_lens.data_ptr() == live_lengths.data_ptr() + torch.testing.assert_close(state.kv_lens, live_lengths) + metadata.mamba_metadata.glm_block_tables = None + with patch("torch.zeros", side_effect=AssertionError("must not allocate fallback tables")): + with pytest.raises(RuntimeError, match="CUDA-graph execution requires"): + backend._cache_state(metadata) + + +@pytest.mark.parametrize("heads", [16, 64]) +def test_backend_output_keeps_flat_contract_without_copy(heads): + storage = torch.randn(3, 64, 512) + per_head = storage[:, :heads] + output = GlmKpoolSparseAttention._finalize_output(None, per_head, None) + assert output.shape == (3, heads * 512) + assert output.untyped_storage().data_ptr() == storage.untyped_storage().data_ptr() + torch.testing.assert_close(output.view(3, heads, 512), per_head) + supplied = torch.empty(3, heads * 512) + assert GlmKpoolSparseAttention._finalize_output(None, per_head, supplied) is supplied + torch.testing.assert_close(supplied, output) + + +@pytest.mark.parametrize( + "kwargs", + [ + {"index_kpool": 0}, + {"index_kpool": 3}, + {"index_topk": 0}, + {"index_topk": 7}, + {"index_always_select_tail": False}, + ], +) +def test_unsupported_pool_layout_fails_before_kernel_launch(kwargs): + with pytest.raises(ValueError, match="glm_kpool requires"): + GlmKpoolSparseParams(**kwargs) + + +def test_derived_geometry_follows_the_checkpoint_defaults(): + params = GlmKpoolSparseParams() + assert params.algorithm == "glm_kpool" + assert params.select_k == 2048 // 4 + # topk positions plus the always-visible incomplete tail (kpool - 1 rows). + assert params.output_width == 2048 + 3 + # Padded to the FlashMLA top-k tile. + assert params.kernel_output_width == 2112 + assert params.packed_state_dim == 2 * 128 + assert params.cache_row_dim == 3 * 128 + small = GlmKpoolSparseParams(index_topk=64, index_kpool=8) + assert (small.select_k, small.output_width, small.kernel_output_width) == (8, 71, 128) + + +def test_paged_slot_indices_returns_page_and_offset_pairs(): + table = torch.tensor([[7, 2, 9], [4, 0, 1]]) + positions = torch.tensor([[0, 5, 16], [3, 8, 23]]) + page, offset = paged_slot_indices(table, positions, tokens_per_block=8) + assert torch.equal(page, torch.tensor([[7, 7, 9], [4, 0, 1]])) + assert torch.equal(offset, torch.tensor([[0, 5, 0], [3, 0, 7]])) + + +def test_positions_to_pool_rows_preserves_sentinels(): + table = torch.tensor([[7, 2], [4, 0]]) + positions = torch.tensor([[0, 9, INDEX_SENTINEL], [15, INDEX_SENTINEL, 8]], dtype=torch.int32) + rows = positions_to_pool_rows(positions, table, 8, base_row=3, rows_per_slot=10) + assert rows.dtype == torch.int32 + # base + slot * rows_per_slot + within_page; sentinels stay -1, never clamped. + assert torch.equal(rows, torch.tensor([[73, 24, -1], [10, -1, 3]], dtype=torch.int32)) + + +def test_latent_pool_rows_reinterprets_a_coalesced_pool_without_copying(): + dim, tpb, slots = 16, 4, 3 + # A wider shared storage: each slot holds this buffer's page plus another + # buffer's payload, so the slot stride exceeds tpb * dim (V2 coalescing). + storage = torch.arange(slots * 2 * tpb * dim, dtype=torch.float32).view(slots, 2 * tpb, dim) + pool = storage[:, :tpb, :] + rows, base_row, rows_per_slot = latent_pool_rows(pool) + assert (base_row, rows_per_slot) == (0, 2 * tpb) + assert rows.data_ptr() == pool.data_ptr() + for slot in range(slots): + for t in range(tpb): + assert torch.equal(rows[base_row + slot * rows_per_slot + t, 0], pool[slot, t]) + # A storage offset that is a whole number of rows is folded into base_row. + shifted = storage.view(-1)[2 * dim :].view(-1, dim)[: slots * tpb].view(slots, tpb, dim) + _, base, per_slot = latent_pool_rows(shifted) + assert (base, per_slot) == (2, tpb) + + +def test_latent_pool_rows_rejects_layouts_without_a_uniform_row_view(): + dim, tpb, slots = 16, 4, 3 + ragged = torch.zeros(slots, tpb, dim + 1)[..., :dim] # rows not contiguous within a page + with pytest.raises(ValueError, match="contiguous within a page"): + latent_pool_rows(ragged) + odd_stride = torch.zeros(slots * tpb * dim + slots * 3).as_strided( + (slots, tpb, dim), (tpb * dim + 3, dim, 1) + ) + with pytest.raises(ValueError, match="not multiples of dim"): + latent_pool_rows(odd_stride) diff --git a/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py b/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py new file mode 100644 index 000000000000..732d22f6a45e --- /dev/null +++ b/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py @@ -0,0 +1,267 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Compare GLM k-pool kernels against PyTorch references using synthetic paged caches.""" + +import pytest +import torch + +from tensorrt_llm._torch.attention.backends.sparse.glm_kpool.kernels import ( + kpool_expand, + kpool_score, + kpool_update, +) + +pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") + +HD = 128 +KPOOL = 4 +TPB = 8 +FP32_MIN = torch.finfo(torch.float32).min + + +def _strided_index_pool(slots: int, gen: torch.Generator) -> torch.Tensor: + """[slots, TPB, 3 * HD] bf16 view with a padded row stride, like the + coalesced V2 pool the backend reads (the kernels take explicit strides).""" + storage = torch.randn(slots, TPB, 3 * HD + 64, generator=gen, device="cuda") + return storage.to(torch.bfloat16)[..., : 3 * HD] + + +def _paged_tables(num_tables: int, pages: int, slots: int, gen: torch.Generator) -> torch.Tensor: + """Disjoint, shuffled slot ids per table (gaps between a request's pages).""" + perm = torch.randperm(slots, generator=gen, device="cuda")[: num_tables * pages] + return perm.view(num_tables, pages).to(torch.int64) + + +def _gather_rows(pool: torch.Tensor, table: torch.Tensor, positions: torch.Tensor) -> torch.Tensor: + page = positions // TPB + return pool[table[page], positions - page * TPB] + + +def _reference_pool_key(pool, table, pos, ape): + """build_pools numerics for the pool containing pos: fp32 softmax + over gate + ape with invisible members masked, bf16 probabilities and + products, fp32 sum, bf16 result.""" + start = pos // KPOOL * KPOOL + members = start + torch.arange(KPOOL, device="cuda") + valid = members <= pos + rows = _gather_rows(pool, table, members) + k, g = rows[:, :HD].float(), rows[:, HD : 2 * HD].float() + logits = (g + ape.float()).masked_fill(~valid[:, None], float("-inf")) + probs = logits.softmax(dim=0).to(torch.bfloat16).float() + prod = (probs * k).to(torch.bfloat16).float() + return prod.sum(dim=0).to(torch.bfloat16) + + +@pytest.mark.parametrize("packed_rows", [False, True], ids=["one_row_per_table", "request_ids"]) +def test_pool_key_refresh_matches_build_pools_numerics(packed_rows): + gen = torch.Generator(device="cuda").manual_seed(0) + slots, pages = 24, 3 + pool = _strided_index_pool(slots, gen) + before = pool.clone() + ape = (torch.randn(KPOOL, HD, generator=gen, device="cuda") * 0.5).to(torch.bfloat16) + if packed_rows: + # Two requests, several rows each: rows of one request share a table. + tables = _paged_tables(2, pages, slots, gen) + request_ids = torch.tensor([0, 0, 0, 1, 1], dtype=torch.int32, device="cuda") + # Distinct pools per request (the backend's contract: one program per + # pool, so no two rows race on one pool key); mid-pool rows exercise + # the invisible-member mask, 6 / 22 the page-crossing addressing. + positions = torch.tensor([1, 7, 9, 6, 22], dtype=torch.int64, device="cuda") + else: + tables = _paged_tables(5, pages, slots, gen) + request_ids = None + positions = torch.tensor([0, 3, 10, 15, 23], dtype=torch.int64, device="cuda") + + kpool_update( + pool, tables, positions, ape, TPB, head_dim=HD, kpool=KPOOL, request_ids=request_ids + ) + torch.cuda.synchronize() + + touched = [] + for row, pos in enumerate(positions.tolist()): + table = tables[request_ids[row].item() if packed_rows else row] + start = pos // KPOOL * KPOOL + slot, off = table[start // TPB].item(), start % TPB + touched.append((slot, off)) + got = pool[slot, off, 2 * HD :] + want = _reference_pool_key(before, table, pos, ape) + torch.testing.assert_close(got.float(), want.float(), atol=1e-2, rtol=1e-2) + # Only the pool-key columns of the first member's row are written. + assert torch.equal(pool[slot, off, : 2 * HD], before[slot, off, : 2 * HD]) + mask = torch.ones(slots, TPB, dtype=torch.bool, device="cuda") + for slot, off in touched: + mask[slot, off] = False + assert torch.equal(pool[mask], before[mask]) + + +def _reference_scores(q, w, pool, tables, kv_lens, *, num_pools_max, q_scale, w_scale): + n = q.shape[0] + out = torch.full((n, num_pools_max), FP32_MIN, device="cuda") + for r in range(n): + num_pools = kv_lens[r].item() // KPOOL + if num_pools == 0: + continue + first = torch.arange(num_pools, device="cuda") * KPOOL + keys = _gather_rows(pool, tables[r], first)[:, 2 * HD :].float() # [P, HD] + scores = torch.relu(q[r].float() @ keys.T * q_scale) # [H, P] + out[r, :num_pools] = (scores * (w[r].float() * w_scale)[:, None]).sum(0) + return out + + +@pytest.mark.parametrize("precision", ["ieee", "tf32"]) +def test_pool_scores_generation_rows(precision): + gen = torch.Generator(device="cuda").manual_seed(1) + n, heads, slots, pages = 6, 4, 64, 8 + pool = _strided_index_pool(slots, gen) + tables = _paged_tables(n, pages, slots, gen) + q = (torch.randn(n, heads, HD, generator=gen, device="cuda")).to(torch.bfloat16) + w = torch.rand(n, heads, generator=gen, device="cuda").to(torch.bfloat16) + # 0 complete pools, partial, exactly full pages, and the capacity edge. + kv_lens = torch.tensor([2, 5, 16, 33, 60, 64], dtype=torch.int64, device="cuda") + num_pools_max = 100 # > 64: two BP blocks, second one entirely invisible + kwargs = dict(num_pools_max=num_pools_max, q_scale=HD**-0.5, w_scale=0.25) + + got = kpool_score( + q, w, pool, tables, kv_lens, TPB, head_dim=HD, kpool=KPOOL, precision=precision, **kwargs + ) + want = _reference_scores(q, w, pool, tables, kv_lens, **kwargs) + visible = want > FP32_MIN + assert torch.equal(got <= FP32_MIN, ~visible) + # bf16 inputs are exact in tf32, so both precisions are fp32 accumulations + # of exact products. + torch.testing.assert_close(got[visible], want[visible], atol=2e-3, rtol=2e-3) + + +def test_pool_scores_packed_context_rows_share_tables(): + """rows_per_program=16 with request_ids: the packed query tokens + of several requests, in position order, including a group straddling two + requests (per-row gather branch).""" + gen = torch.Generator(device="cuda").manual_seed(2) + heads, slots, pages = 3, 64, 8 + pool = _strided_index_pool(slots, gen) + tables = _paged_tables(2, pages, slots, gen) + # Request 0: 21 query tokens (positions 0..20), request 1: 19 (0..18). + lens = [21, 19] + request_ids = torch.cat( + [torch.full((length,), i, dtype=torch.int32) for i, length in enumerate(lens)] + ).cuda() + kv_lens = torch.cat([torch.arange(1, length + 1) for length in lens]).cuda() + n = kv_lens.shape[0] + q = torch.randn(n, heads, HD, generator=gen, device="cuda").to(torch.bfloat16) + w = torch.rand(n, heads, generator=gen, device="cuda").to(torch.bfloat16) + kwargs = dict(num_pools_max=8, q_scale=HD**-0.5, w_scale=1.0) + + got = kpool_score( + q, + w, + pool, + tables, + kv_lens, + TPB, + head_dim=HD, + kpool=KPOOL, + rows_per_program=16, + request_ids=request_ids, + **kwargs, + ) + want = _reference_scores(q, w, pool, tables[request_ids.long()], kv_lens, **kwargs) + visible = want > FP32_MIN + assert torch.equal(got <= FP32_MIN, ~visible) + torch.testing.assert_close(got[visible], want[visible], atol=2e-3, rtol=2e-3) + + +def test_pool_scores_broadcast_table_single_context_request(): + """One context request: a stride-0 broadcast table with rows_per_program=16.""" + gen = torch.Generator(device="cuda").manual_seed(3) + heads, slots, pages, n = 4, 32, 4, 30 + pool = _strided_index_pool(slots, gen) + table = _paged_tables(1, pages, slots, gen) + tables = table.expand(n, pages) + assert tables.stride(0) == 0 + kv_lens = torch.arange(1, n + 1, device="cuda") + q = torch.randn(n, heads, HD, generator=gen, device="cuda").to(torch.bfloat16) + w = torch.rand(n, heads, generator=gen, device="cuda").to(torch.bfloat16) + kwargs = dict(num_pools_max=8, q_scale=HD**-0.5, w_scale=1.0) + + got = kpool_score( + q, w, pool, tables, kv_lens, TPB, head_dim=HD, kpool=KPOOL, rows_per_program=16, **kwargs + ) + want = _reference_scores(q, w, pool, tables, kv_lens, **kwargs) + visible = want > FP32_MIN + assert torch.equal(got <= FP32_MIN, ~visible) + torch.testing.assert_close(got[visible], want[visible], atol=2e-3, rtol=2e-3) + + with pytest.raises(ValueError, match="rows_per_program"): + kpool_score( + q, + w, + pool, + table.repeat(n, 1), + kv_lens, + TPB, + head_dim=HD, + kpool=KPOOL, + rows_per_program=16, + **kwargs, + ) + + +def test_pool_expansion_graph_replay_tracks_request_tables_and_tails(): + # Requests occupy disjoint slots, with gaps between slots from coalesced buffers. + tables = torch.tensor([[3, 1], [5, 2]], dtype=torch.int64, device="cuda") + requests = torch.tensor([0, 0, 1, 1], dtype=torch.int32, device="cuda") + lengths = torch.tensor([3, 8, 9, 15], dtype=torch.int64, device="cuda") + selected = torch.tensor([[0, -1], [1, 0], [1, 0], [2, 1]], dtype=torch.int32, device="cuda") + + def expand(): + return kpool_expand( + selected, + lengths, + tables, + 8, + base_row=2, + rows_per_slot=24, + kpool=4, + out_width=64, + request_ids=requests, + ) + + def reference(): + result = torch.full((4, 64), -1, dtype=torch.int32) + for row, (request, length, pools) in enumerate( + zip(requests.cpu().tolist(), lengths.cpu().tolist(), selected.cpu().tolist()) + ): + positions = [ + pool * 4 + member if 0 <= pool < length // 4 else -1 + for pool in pools + for member in range(4) + ] + positions += list(range(length // 4 * 4, length)) + for col, position in enumerate(positions): + if position >= 0: + result[row, col] = 2 + int(tables[request, position // 8]) * 24 + position % 8 + return result.cuda() + + expand() + torch.cuda.synchronize() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + output = expand() + graph.replay() + assert torch.equal(output, reference()) + lengths.copy_(torch.tensor([4, 7, 12, 16], device="cuda")) + tables.copy_(tables.flip(0)) + graph.replay() + assert torch.equal(output, reference()) diff --git a/tests/unittest/_torch/modeling/test_glm5_next_contracts.py b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py new file mode 100644 index 000000000000..23bd74dcfd2c --- /dev/null +++ b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py @@ -0,0 +1,331 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GLM configuration, loading and vision ownership regressions without checkpoint files.""" + +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch +from transformers import PretrainedConfig + +from tensorrt_llm._torch.distributed import AllReduce +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.models.checkpoints.hf.glm5_next_weight_mapper import ( + Disposition, + audit_glm5_next_checkpoint, +) +from tensorrt_llm._torch.models.modeling_glm5_next import ( + Glm5NextForCausalLM, + Glm5NextLinearAttention, + Glm5NextSparseAttention, +) +from tensorrt_llm._torch.models.modeling_glm5_next_vision import Glm5NextVisionModelBase +from tensorrt_llm._torch.pyexecutor.config_utils import get_glm5_next_layer_masks +from tensorrt_llm.mapping import Mapping + + +def _config(): + config = PretrainedConfig( + model_type="glm5_next", + text_config=PretrainedConfig( + model_type="glm5_next_text", + dtype="bfloat16", + rms_norm_eps=1e-05, + num_attention_heads=64, + q_lora_rank=1536, + kv_lora_rank=512, + qk_nope_head_dim=256, + qk_rope_head_dim=0, + v_head_dim=256, + index_n_heads=32, + index_head_dim=128, + index_topk=2048, + index_kpool=4, + index_kpool_always_select_tail=True, + hidden_size=256, + num_hidden_layers=2, + mlp_layer_types=["dense", "dense"], + first_k_dense_replace=2, + num_nextn_predict_layers=1, + max_position_embeddings=1024, + linear_attn_config={ + "num_heads": 64, + "head_dim": 128, + "short_conv_kernel_size": 4, + "gate_lower_bound": -5.0, + }, + ), + vision_config=PretrainedConfig( + attention_bias=True, + rms_norm_eps=1e-05, + swiglu_limit=7.0, + in_channels=3, + depth=1, + hidden_size=256, + num_heads=4, + intermediate_size=512, + out_hidden_size=256, + projection_intermediate_size=512, + patch_size=2, + temporal_patch_size=2, + spatial_merge_size=2, + ), + ) + # The shared Transformers version predates this layer name. Attach the + # synthetic schedule after its constructor validates the base config. + config.text_config.layer_types = ["linear_attention", "deepseek_sparse_attention"] + return config + + +@pytest.mark.cpu_only +def test_layer_masks_accept_composite_and_text_configs(): + config = _config() + expected = ([False, True], [True, False]) + assert get_glm5_next_layer_masks(config) == expected + assert get_glm5_next_layer_masks(config.text_config) == expected + config.text_config.layer_types.pop() + with pytest.raises(ValueError, match="num_hidden_layers"): + get_glm5_next_layer_masks(config) + config.text_config.layer_types = ["linear_attention", "unknown"] + with pytest.raises(ValueError, match="must be exactly one"): + get_glm5_next_layer_masks(config) + + +@pytest.mark.cpu_only +def test_checkpoint_routes_vision_and_optional_mtp_separately(): + config = _config() + keys = [ + "model.visual.patch_embed.proj.weight", + "model.language_model.layers.0.self_attn.A_log", + "model.language_model.layers.2.eh_proj.weight", + "lm_head.weight", + "unexpected.weight", + ] + plain = audit_glm5_next_checkpoint(keys, config) + mtp = audit_glm5_next_checkpoint(keys, config, num_mtp_layers=1) + assert plain.disposition[keys[0]] == Disposition.IGNORED + assert plain.disposition[keys[2]] == Disposition.IGNORED + assert mtp.destinations["model.layers.2.eh_proj.weight"] == keys[2] + assert mtp.destinations["model.layers.0.self_attn.A_log"] == keys[1] + assert mtp.unresolved == ["unexpected.weight"] + for invalid in (-1, 2): + with pytest.raises(ValueError, match=f"cannot load {invalid} MTP layers"): + audit_glm5_next_checkpoint(keys, config, num_mtp_layers=invalid) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("route", ["valid", "py_mamba", "manager_preference", "v1", "cpp", "ucx"]) +def test_cache_manager_routing_guards(route, monkeypatch): + from tensorrt_llm._torch.attention.backends.sparse.glm_kpool import Glm5NextCacheManager + from tensorrt_llm._torch.pyexecutor._util import get_kv_cache_manager_cls + from tensorrt_llm.llmapi import CacheTransceiverConfig, KvCacheConfig + + monkeypatch.delenv("TRTLLM_USE_PY_MAMBA", raising=False) + monkeypatch.delenv("TLLM_MAMBA_MANAGER_PREFERENCE", raising=False) + if route == "py_mamba": + monkeypatch.setenv("TRTLLM_USE_PY_MAMBA", "1") + if route == "manager_preference": + monkeypatch.setenv("TLLM_MAMBA_MANAGER_PREFERENCE", "MIXED") + config = ModelConfig(pretrained_config=_config()) + kv = KvCacheConfig(use_kv_cache_manager_v2=route != "v1") + transceiver = CacheTransceiverConfig( + backend="UCX" if route == "ucx" else "NIXL", + transceiver_runtime="CPP" if route == "cpp" else "PYTHON", + ) + if route == "valid": + assert get_kv_cache_manager_cls(config, kv) is Glm5NextCacheManager + assert get_kv_cache_manager_cls(config, kv, True, transceiver) is Glm5NextCacheManager + else: + with pytest.raises(ValueError, match="glm5_next"): + get_kv_cache_manager_cls(config, kv, True, transceiver) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("deferred", [False, True]) +def test_encoder_only_factory_materializes_attention_weights(deferred): + from tensorrt_llm._torch.models.modeling_auto import AutoModelForCausalLM + from tensorrt_llm._torch.models.modeling_glm5_next_vision import Glm5NextVisionModel + + config = _config() + config.architectures = ["Glm5NextForConditionalGeneration"] + model_config = ModelConfig( + pretrained_config=config, mm_encoder_only=True, skip_create_weights_in_init=deferred + ) + with torch.device("meta"): + encoder = AutoModelForCausalLM.from_config(model_config) + assert isinstance(encoder, Glm5NextVisionModelBase) + assert isinstance(encoder.visual, Glm5NextVisionModel) + for block in encoder.visual.blocks: + for projection in (block.attn.qkv_proj, block.attn.o_proj, block.mlp.down_proj): + assert projection._weights_created + assert projection.weight is not None + + +@pytest.mark.cpu_only +def test_attention_dp_speculation_rejected_before_model_construction(): + config = SimpleNamespace( + pretrained_config=_config(), + mapping=Mapping(world_size=4, tp_size=4, enable_attention_dp=True), + spec_config=object(), + ) + with ( + patch( + "tensorrt_llm._torch.models.modeling_glm5_next.Glm5NextModel", + side_effect=AssertionError("must reject before constructing layers"), + ), + pytest.raises(ValueError, match="does not support attention DP"), + ): + Glm5NextForCausalLM(config) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +def test_kda_shards_preserve_fp32_gate_parameters(): + config = _config().text_config + full_bias = torch.arange(64 * 128, dtype=torch.float32) + full_log = torch.arange(64, dtype=torch.float32) + biases, logs = [], [] + with patch("tensorrt_llm._torch.modules.kimi_kda.kimi_kda_mixer.AllReduce"): + for rank in range(4): + layer = Glm5NextLinearAttention( + config, 0, mapping=Mapping(world_size=4, tp_size=4, rank=rank) + ) + assert not layer.use_full_rank_gate + assert layer.A_log.shape == (16,) + assert layer.dt_bias.shape == (16 * 128,) + assert layer.A_log.dtype == layer.dt_bias.dtype == torch.float32 + biases.append(layer.shard_checkpoint_tensor("dt_bias", full_bias)) + logs.append(layer.shard_checkpoint_tensor("A_log", full_log)) + torch.testing.assert_close(torch.cat(biases), full_bias) + torch.testing.assert_close(torch.cat(logs), full_log) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@torch.inference_mode() +def test_vision_attention_dp_is_local_and_matches_single_rank(): + config = _config() + reference = Glm5NextVisionModelBase(ModelConfig(pretrained_config=config)).cuda().eval() + reference.visual.setup_attn_metadata(max_num_tokens=128) + torch.manual_seed(7) + for parameter in reference.parameters(): + parameter.normal_(0, 0.05) + + # Rank 0 has no image; the other ranks see different token counts and + # pixels. Any TP collective on the encoder path is an error, even when + # another rank has no encoder work at all. + def local_allreduce(**kwargs): + assert kwargs["mapping"].tp_size == 1, "vision must not reduce across DP ranks" + return AllReduce(**kwargs) + + with patch("tensorrt_llm._torch.distributed.AllReduce", side_effect=local_allreduce): + for rank, width in enumerate((0, 4, 6, 8)): + mapping = Mapping(world_size=4, tp_size=4, rank=rank, enable_attention_dp=True) + local = ( + Glm5NextVisionModelBase(ModelConfig(pretrained_config=config, mapping=mapping)) + .cuda() + .eval() + ) + local.load_state_dict(reference.state_dict()) + local.visual.setup_attn_metadata(max_num_tokens=128) + if not width: + continue + pixels = torch.randn(4 * width, 3 * 2 * 2 * 2, device="cuda", dtype=torch.bfloat16) + grid = torch.tensor([[1, 4, width]]) + expected = reference.encode_batched(pixels, grid) + actual = local.encode_batched(pixels, grid) + torch.testing.assert_close(actual, expected, rtol=0, atol=0) + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("temporal_groups", [1, 3], ids=["image", "video"]) +@torch.inference_mode() +def test_vision_weights_and_features_match_hf(temporal_groups): + hf_module = pytest.importorskip("transformers.models.glm5_next.modeling_glm5_next") + Glm5NextVisionModel = hf_module.Glm5NextVisionModel + + from tensorrt_llm._torch.models.modeling_glm5_next_vision import _flatten_video_grid_thw + + torch.manual_seed(11) + config = _config() + config.vision_config = hf_module.Glm5NextVisionConfig.from_dict(config.vision_config.to_dict()) + hf = Glm5NextVisionModel(config.vision_config).to(device="cuda", dtype=torch.bfloat16).eval() + runtime = Glm5NextVisionModelBase(ModelConfig(pretrained_config=config)).cuda().eval() + runtime.visual.setup_attn_metadata(max_num_tokens=128) + weights = {"model.visual." + name: tensor for name, tensor in hf.state_dict().items()} + runtime.load_weights(weights) + pixels = torch.randn( + temporal_groups * 4 * 6, 3 * 2 * 2 * 2, device="cuda", dtype=torch.bfloat16 + ) + grid = torch.tensor([[temporal_groups, 4, 6]]) + flat_grid = _flatten_video_grid_thw(grid) + expected = hf(pixels, grid_thw=flat_grid.cuda()).pooler_output + actual = runtime.encode_batched(pixels, flat_grid) + assert actual.shape == (temporal_groups * 6, config.vision_config.out_hidden_size) + assert torch.isfinite(actual).all() + cosine = torch.nn.functional.cosine_similarity( + actual.float().flatten(), expected.float().flatten(), dim=0 + ) + relative_l2 = (actual.float() - expected.float()).norm() / expected.float().norm().clamp_min( + 1e-12 + ) + assert cosine > 0.999 + assert relative_l2 < 0.03 + + +@pytest.mark.skipif( + not torch.cuda.is_available() or torch.cuda.get_device_capability(0)[0] != 10, + reason="requires Blackwell sparse MLA", +) +@torch.no_grad() +def test_sparse_prefill_continuation_preserves_partial_pools(): + """A chunk boundary inside a k-pool preserves both cached state and attention outputs.""" + torch.manual_seed(17) + layer = Glm5NextSparseAttention(_config().text_config, 1).cuda().eval() + for parameter in layer.parameters(): + parameter.normal_(0, 0.05) + hidden = torch.randn(13, 256, device="cuda", dtype=torch.bfloat16) + hidden[:5] += 1 + tables = torch.zeros(1, 256, device="cuda", dtype=torch.long) + tables[0, :2] = torch.tensor([2, 0], device="cuda") + + def run(lengths, discard_prefix=False): + latent = torch.zeros(4, 8, 1, layer.kv_lora_rank, device="cuda", dtype=torch.bfloat16) + index = torch.zeros( + 4, 8, 1, layer.indexer.cache_state_dim, device="cuda", dtype=torch.bfloat16 + ) + manager = SimpleNamespace( + tokens_per_block=8, + get_latent_state_buffer=lambda _: latent, + get_index_state_buffer=lambda _: index, + ) + outputs = [] + cached = 0 + for length in lengths: + if cached and discard_prefix: + latent.zero_() + index.zero_() + metadata = SimpleNamespace( + kv_cache_manager=manager, + seq_lens=torch.tensor([length]), + num_contexts=1, + kv_lens_cuda=torch.tensor([cached + length], device="cuda", dtype=torch.int32), + mamba_metadata=SimpleNamespace(glm_block_tables=tables), + ) + outputs.append( + layer.forward_prefill( + hidden[cached : cached + length], [0, length], [cached], metadata + ) + ) + cached += length + return torch.cat(outputs), latent, index + + reference, latent, index = run([13]) + actual, chunked_latent, chunked_index = run([5, 8]) + torch.testing.assert_close(chunked_latent, latent, rtol=0.02, atol=1e-3) + torch.testing.assert_close(chunked_index, index, rtol=0.02, atol=1e-3) + relative_l2 = (actual.float() - reference.float()).norm() / reference.float().norm() + assert relative_l2 < 0.03 + # Negative control: this gate must detect losing the previous chunk. + broken, _, _ = run([5, 8], discard_prefix=True) + broken_l2 = (broken.float() - reference.float()).norm() / reference.float().norm() + assert broken_l2 > 0.03 diff --git a/tests/unittest/_torch/models/checkpoints/hf/test_weight_loader.py b/tests/unittest/_torch/models/checkpoints/hf/test_weight_loader.py index ccf1973bb7f2..1ebd6a231336 100644 --- a/tests/unittest/_torch/models/checkpoints/hf/test_weight_loader.py +++ b/tests/unittest/_torch/models/checkpoints/hf/test_weight_loader.py @@ -496,7 +496,8 @@ def test_prefetch_files_emits_progress_heartbeat(tmp_path, monkeypatch): assert len(progress_logs) >= 12 -def test_kimi_k3_lazy_load_records_the_checkpoint_dir(tmp_path): +@pytest.mark.parametrize("model_type", ["kimi_k3", "glm5_next", "glm5_next_text"]) +def test_lazy_load_records_the_checkpoint_dir(tmp_path, model_type): """A model that re-opens shards itself needs the directory back. Kimi K3 streams its rank-local experts per shard file to avoid holding @@ -509,16 +510,17 @@ def test_kimi_k3_lazy_load_records_the_checkpoint_dir(tmp_path): import safetensors.torch import torch - (tmp_path / "config.json").write_text(json.dumps({"model_type": "kimi_k3"})) - safetensors.torch.save_file( - {"w": torch.zeros(2, 2)}, tmp_path / "model-00001-of-00001.safetensors" - ) + (tmp_path / "config.json").write_text(json.dumps({"model_type": model_type})) + stored = torch.arange(6, dtype=torch.float32).reshape(2, 3) + safetensors.torch.save_file({"w": stored}, tmp_path / "model-00001-of-00001.safetensors") loader = HfWeightLoader() try: weights = loader.load_weights(str(tmp_path), Mapping()) assert isinstance(weights, ConsumableWeightsDict) assert weights.checkpoint_dir == str(tmp_path) + assert not torch.is_tensor(weights["w"]) + torch.testing.assert_close(weights["w"][:], stored, rtol=0, atol=0) finally: loader.cleanup() diff --git a/tests/unittest/_torch/modules/kimi_kda/test_kda_decode_op.py b/tests/unittest/_torch/modules/kimi_kda/test_kda_decode_op.py index 0f8e9beb29a0..df46ae1e4c84 100644 --- a/tests/unittest/_torch/modules/kimi_kda/test_kda_decode_op.py +++ b/tests/unittest/_torch/modules/kimi_kda/test_kda_decode_op.py @@ -46,14 +46,17 @@ def _has_supported_gpu() -> bool: def _make_attention_pair( - *, finalize_decode_weights: bool = True + *, + finalize_decode_weights: bool = True, + num_heads: int = NUM_HEADS, + use_full_rank_gate: bool = True, ) -> tuple[KimiKDALinearAttention, KimiKDAReference]: common = { "hidden_size": HIDDEN_SIZE, - "num_heads": NUM_HEADS, + "num_heads": num_heads, "head_dim": HEAD_DIM, "conv_kernel_size": CONV_KERNEL_SIZE, - "use_full_rank_gate": True, + "use_full_rank_gate": use_full_rank_gate, "gate_lower_bound": -5.0, "rms_norm_eps": 1e-5, "dtype": torch.bfloat16, @@ -62,7 +65,7 @@ def _make_attention_pair( hidden_size=HIDDEN_SIZE, rms_norm_eps=common["rms_norm_eps"], linear_attn_config={ - "num_heads": NUM_HEADS, + "num_heads": num_heads, "head_dim": HEAD_DIM, "short_conv_kernel_size": CONV_KERNEL_SIZE, "use_full_rank_gate": common["use_full_rank_gate"], @@ -81,8 +84,10 @@ def _make_attention_pair( return optimized, reference -def _make_cache(batch_size: int = BATCH_SIZE) -> KimiKDATestCachedState: - projection_size = NUM_HEADS * HEAD_DIM +def _make_cache( + batch_size: int = BATCH_SIZE, *, num_heads: int = NUM_HEADS +) -> KimiKDATestCachedState: + projection_size = num_heads * HEAD_DIM return KimiKDATestCachedState( conv_state_q=( torch.randn( @@ -117,7 +122,7 @@ def _make_cache(batch_size: int = BATCH_SIZE) -> KimiKDATestCachedState: recurrent_state=( torch.randn( batch_size, - NUM_HEADS, + num_heads, HEAD_DIM, HEAD_DIM, dtype=torch.float32, @@ -140,7 +145,7 @@ def _run_production_decode( include_metadata: bool = True, ) -> tuple[torch.Tensor, KimiKDATestCachedState]: batch_size = hidden_states.shape[0] - projection_size = NUM_HEADS * HEAD_DIM + projection_size = attention.proj_size if slot_indices is None: slot_indices = torch.arange(batch_size, device="cuda", dtype=torch.long) if conv_pool is None: @@ -202,11 +207,37 @@ def _assert_close(actual: torch.Tensor, expected: torch.Tensor) -> None: assert relative_l2 < 3e-2 +@torch.no_grad() +@pytest.mark.parametrize("num_heads", [16, 64]) +def test_low_rank_gate_projections_match_unfused_linears(num_heads: int) -> None: + """Check each gate before nonlinearities can hide projection-layout errors.""" + torch.manual_seed(31) + optimized, reference = _make_attention_pair(num_heads=num_heads, use_full_rank_gate=False) + hidden = torch.randn(2, 5, HIDDEN_SIZE, device="cuda", dtype=torch.bfloat16) + beta, forget_gate, output_gate = optimized._project_gate_inputs(hidden) + expected = ( + reference.b_proj(hidden), + reference.f_b_proj(reference.f_a_proj(hidden)), + reference.g_b_proj(reference.g_a_proj(hidden)), + ) + for actual, wanted in zip((beta, forget_gate, output_gate), expected): + torch.testing.assert_close(actual, wanted, rtol=0.02, atol=0.002) + + @torch.no_grad() @pytest.mark.parametrize("batch_size", [1, BATCH_SIZE]) -def test_optimized_decode_matches_fla_reference(batch_size: int) -> None: +@pytest.mark.parametrize( + ("num_heads", "use_full_rank_gate"), + [(NUM_HEADS, True), (16, False), (64, False)], + ids=["full-rank", "low-rank-h16", "low-rank-h64"], +) +def test_optimized_decode_matches_fla_reference( + batch_size: int, num_heads: int, use_full_rank_gate: bool +) -> None: torch.manual_seed(0) - optimized, reference = _make_attention_pair() + optimized, reference = _make_attention_pair( + num_heads=num_heads, use_full_rank_gate=use_full_rank_gate + ) hidden_states = ( torch.randn( batch_size, @@ -217,11 +248,15 @@ def test_optimized_decode_matches_fla_reference(batch_size: int) -> None: ) * 0.05 ) - initial_cache = _make_cache(batch_size) + initial_cache = _make_cache(batch_size, num_heads=num_heads) actual_output, actual_cache = _run_production_decode( - optimized, hidden_states, copy.deepcopy(initial_cache) + optimized, + hidden_states, + copy.deepcopy(initial_cache), + ssm_state_indices=torch.arange(batch_size, device="cuda", dtype=torch.int32), ) + assert optimized._o_dense is not None, "decode must exercise the fused kernel" expected_output, expected_cache = reference.forward_decode( hidden_states, copy.deepcopy(initial_cache) ) diff --git a/tests/unittest/_torch/modules/kimi_kda/test_kda_prefill_op.py b/tests/unittest/_torch/modules/kimi_kda/test_kda_prefill_op.py index 6dfca6101def..bf141d7a6306 100644 --- a/tests/unittest/_torch/modules/kimi_kda/test_kda_prefill_op.py +++ b/tests/unittest/_torch/modules/kimi_kda/test_kda_prefill_op.py @@ -49,15 +49,18 @@ def _has_supported_gpu() -> bool: def _make_kda( expected_prefill_kernel_path: str = "optimized", source_state_dict: Mapping[str, torch.Tensor] | None = None, + *, + num_heads: int = NUM_HEADS, + use_full_rank_gate: bool = True, ) -> KimiKDALinearAttention: cfg = SimpleNamespace( hidden_size=HIDDEN_SIZE, rms_norm_eps=1e-5, linear_attn_config={ - "num_heads": NUM_HEADS, + "num_heads": num_heads, "head_dim": HEAD_DIM, "short_conv_kernel_size": CONV_KERNEL_SIZE, - "use_full_rank_gate": True, + "use_full_rank_gate": use_full_rank_gate, "gate_lower_bound": -5.0, }, ) @@ -72,13 +75,18 @@ def _make_kda( return kda -def _make_reference(source_state_dict: Mapping[str, torch.Tensor]) -> KimiKDAReference: +def _make_reference( + source_state_dict: Mapping[str, torch.Tensor], + *, + num_heads: int = NUM_HEADS, + use_full_rank_gate: bool = True, +) -> KimiKDAReference: reference = KimiKDAReference( hidden_size=HIDDEN_SIZE, - num_heads=NUM_HEADS, + num_heads=num_heads, head_dim=HEAD_DIM, conv_kernel_size=CONV_KERNEL_SIZE, - use_full_rank_gate=True, + use_full_rank_gate=use_full_rank_gate, gate_lower_bound=-5.0, rms_norm_eps=1e-5, dtype=torch.bfloat16, @@ -103,7 +111,7 @@ def _run_production_prefill( else: batch_size = cu_seqlens.numel() - 1 - projection_size = NUM_HEADS * HEAD_DIM + projection_size = attention.proj_size if conv_pool is None: conv_pool = torch.zeros( batch_size, @@ -115,7 +123,7 @@ def _run_production_prefill( if state_pool is None: state_pool = torch.zeros( batch_size, - NUM_HEADS, + attention.num_heads, HEAD_DIM, HEAD_DIM, dtype=torch.float32, @@ -320,10 +328,17 @@ def replay_cache() -> torch.Tensor: @torch.no_grad() -def test_optimized_prefill_matches_fla_reference() -> None: +@pytest.mark.parametrize( + ("num_heads", "use_full_rank_gate"), + [(NUM_HEADS, True), (16, False), (64, False)], + ids=["full-rank", "low-rank-h16", "low-rank-h64"], +) +def test_optimized_prefill_matches_fla_reference(num_heads: int, use_full_rank_gate: bool) -> None: torch.manual_seed(0) - optimized = _make_kda() - reference = _make_reference(optimized.state_dict()) + optimized = _make_kda(num_heads=num_heads, use_full_rank_gate=use_full_rank_gate) + reference = _make_reference( + optimized.state_dict(), num_heads=num_heads, use_full_rank_gate=use_full_rank_gate + ) # Keep B=2 across a T transition: eqlen mBeta/mAqk/mAkk batch strides # depend on T and therefore require distinct compiled kernel variants. diff --git a/tests/unittest/_torch/modules/kimi_kda/test_kimi_kda_fused_verify_parity.py b/tests/unittest/_torch/modules/kimi_kda/test_kimi_kda_fused_verify_parity.py index 805a4f864217..273395f9b7a8 100644 --- a/tests/unittest/_torch/modules/kimi_kda/test_kimi_kda_fused_verify_parity.py +++ b/tests/unittest/_torch/modules/kimi_kda/test_kimi_kda_fused_verify_parity.py @@ -26,7 +26,7 @@ recorded between rounds. Identical hidden states are fed to both worlds; the fused world additionally -uses fused QKVG and [f_a|b] projections with multi-stream overlap. With mixed +uses fused QKV(G) and full- or low-rank gate projections with multi-stream overlap. With mixed per-request acceptance between rounds, matching round-2 outputs proves the projection fusion and replay bookkeeping (shifted ``cu_seqlens`` layout, conv-window seeding, pending-count plumbing) reproduce the promoted-state @@ -78,7 +78,7 @@ def _is_blackwell(): @torch.no_grad() -def _make_runtime(seed, aux_stream=None, checkpoint_fp8=False): +def _make_runtime(seed, aux_stream=None, checkpoint_fp8=False, *, num_heads=H, use_full_rank_gate=True): # A real KimiLinearConfig (not a SimpleNamespace) so the runtime sees the # same config surface it does in production. ``linear_attn_config`` carries # the per-layer KDA params the runtime reads plus the (unused here) @@ -89,10 +89,10 @@ def _make_runtime(seed, aux_stream=None, checkpoint_fp8=False): linear_attn_config=dict( kda_layers=[1], full_attn_layers=[], - num_heads=H, + num_heads=num_heads, head_dim=K, short_conv_kernel_size=W, - use_full_rank_gate=True, + use_full_rank_gate=use_full_rank_gate, gate_lower_bound=LB, ), ) @@ -134,22 +134,22 @@ def _make_runtime(seed, aux_stream=None, checkpoint_fp8=False): return rt -def _make_pools(B, seed): +def _make_pools(B, seed, *, num_heads=H): gen = torch.Generator(device="cuda").manual_seed(seed) - d = H * K + d = num_heads * K conv_pool = ( torch.randn(B, 3 * d, W - 1, generator=gen, device="cuda", dtype=torch.float32) * 0.5 ).to(torch.bfloat16) - ssm_pool = torch.randn(B, H, K, K, generator=gen, device="cuda", dtype=torch.float32) + ssm_pool = torch.randn(B, num_heads, K, K, generator=gen, device="cuda", dtype=torch.float32) ssm_pool *= torch.linspace(0.5, 1.5, K, device="cuda").view(1, 1, K, 1) return conv_pool, ssm_pool -def _make_fused_layer_cache(B, conv_pool): +def _make_fused_layer_cache(B, conv_pool, *, num_heads=H): """Replay caches shaped like PythonMambaCacheManager's KDA allocation, with the committed conv window seeded from the base pool (the prefill seeding contract: the base pool stores committed columns directly).""" - d = H * K + d = num_heads * K S = W - 1 + M def _conv_cache(section): @@ -163,7 +163,7 @@ def _conv_cache(section): kda_conv_v=_conv_cache(2), kda_qkg_cache=torch.zeros(B, M, 3, d, device="cuda", dtype=torch.float32), kda_v_cache=torch.zeros(B, M, d, device="cuda", dtype=torch.float32), - kda_beta_cache=torch.zeros(B, M, H, device="cuda", dtype=torch.float32), + kda_beta_cache=torch.zeros(B, M, num_heads, device="cuda", dtype=torch.float32), prev_num_accepted_tokens=torch.zeros(B, dtype=torch.int32, device="cuda"), has_kda_replay_caches=True, intermediate_conv_window=None, @@ -171,15 +171,15 @@ def _conv_cache(section): ) -def _make_seq_layer_cache(B): - d = H * K +def _make_seq_layer_cache(B, *, num_heads=H): + d = num_heads * K return SimpleNamespace( kda_qkg_cache=None, has_kda_replay_caches=False, intermediate_conv_window=torch.zeros( B, M + 1, 3 * d, W - 1, device="cuda", dtype=torch.bfloat16 ), - intermediate_ssm=torch.zeros(B, M + 1, H, K, K, device="cuda", dtype=torch.float32), + intermediate_ssm=torch.zeros(B, M + 1, num_heads, K, K, device="cuda", dtype=torch.float32), ) @@ -199,16 +199,26 @@ def _rep(name, a, b): return cos > 0.999 and rel < 3e-2 -@pytest.mark.parametrize("checkpoint_fp8", [False, True]) @torch.no_grad() -def test_fused_vs_sequential_two_rounds(checkpoint_fp8): +@pytest.mark.parametrize( + ("num_heads", "use_full_rank_gate", "checkpoint_fp8"), + [(H, True, False), (H, True, True), (16, False, False), (64, False, False)], + ids=["full-rank-bf16", "full-rank-fp8", "low-rank-h16", "low-rank-h64"], +) +def test_fused_vs_sequential_two_rounds(num_heads, use_full_rank_gate, checkpoint_fp8): from tensorrt_llm._torch.modules.multi_stream_utils import with_multi_stream torch.manual_seed(0) B = 4 T = M + 1 - rt_seq = _make_runtime(seed=1, checkpoint_fp8=checkpoint_fp8) - rt_fused = _make_runtime(seed=1, aux_stream=torch.cuda.Stream(), checkpoint_fp8=checkpoint_fp8) + rt_seq = _make_runtime( + seed=1, checkpoint_fp8=checkpoint_fp8, num_heads=num_heads, + use_full_rank_gate=use_full_rank_gate, + ) + rt_fused = _make_runtime( + seed=1, aux_stream=torch.cuda.Stream(), checkpoint_fp8=checkpoint_fp8, + num_heads=num_heads, use_full_rank_gate=use_full_rank_gate, + ) rt_fused.finalize_decode_weights() if checkpoint_fp8: from tensorrt_llm._torch.modules.linear import Linear @@ -224,11 +234,11 @@ def test_fused_vs_sequential_two_rounds(checkpoint_fp8): assert rt_fused._bfa_proj_weight is not None slot_indices = torch.arange(B, dtype=torch.int32, device="cuda") - conv_pool_seq, ssm_pool_seq = _make_pools(B, seed=2) + conv_pool_seq, ssm_pool_seq = _make_pools(B, seed=2, num_heads=num_heads) conv_pool_fused = conv_pool_seq.clone() ssm_pool_fused = ssm_pool_seq.clone() - cache_seq = _make_seq_layer_cache(B) - cache_fused = _make_fused_layer_cache(B, conv_pool_fused) + cache_seq = _make_seq_layer_cache(B, num_heads=num_heads) + cache_fused = _make_fused_layer_cache(B, conv_pool_fused, num_heads=num_heads) gen = torch.Generator(device="cuda").manual_seed(3) @@ -266,7 +276,7 @@ def tokens(scale=0.5): x2, T, cache_seq, conv_pool_seq, ssm_pool_seq, slot_indices ) ) - core2_fused = x2.new_empty(B * T, H, K) + core2_fused = x2.new_empty(B * T, num_heads, K) with with_multi_stream(True): result2_fused = rt_fused.forward_verify( x2, From 439f801fb12af1e0229d1bc164f564b2c859e064 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Mon, 14 Sep 2026 09:14:43 -0700 Subject: [PATCH 02/35] [None][fix] Validate GLM FP8 KV cache and sparse forward contracts Signed-off-by: Ruocheng Jia --- tensorrt_llm/_torch/pyexecutor/_util.py | 4 + .../sparse/glm_kpool/test_glm_kpool.py | 147 +++++++++++++++++- .../modeling/test_glm5_next_contracts.py | 51 ++++++ 3 files changed, 201 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index 35ad04c9e5b4..c6763fdb8d67 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -3009,6 +3009,10 @@ def _create_kv_cache_manager( # Glm5NextCacheManager. Must come before the is_mla(...) route: the # glm5_next text config carries MLA fields, but only 11 of its 45 # layers are sparse MLA. + if kv_cache_dtype == tensorrt_llm.bindings.DataType.FP8: + raise ValueError( + "glm5_next does not support FP8 KV cache; use " + "kv_cache_config.dtype='auto' with a BF16 latent cache.") if max_beam_width > 1: raise ValueError("glm5_next + beam search is not supported yet.") if not estimating_kv_cache and kv_connector_manager is not None: diff --git a/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py b/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py index e0f3a0104b1c..400923f2c556 100644 --- a/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py +++ b/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py @@ -15,11 +15,15 @@ """CPU tests for GLM sparse attention metadata and cache layouts.""" from types import SimpleNamespace -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest import torch +from tensorrt_llm._torch.attention.backends.interface import ( + AttentionForwardArgs, + AttentionInputType, +) from tensorrt_llm._torch.attention.backends.sparse.glm_kpool import ( INDEX_SENTINEL, GlmKpoolSparseAttention, @@ -28,10 +32,151 @@ paged_slot_indices, positions_to_pool_rows, ) +from tensorrt_llm._torch.attention.backends.sparse.glm_kpool.params import ( + GlmKpoolBackendForwardArgs, +) pytestmark = pytest.mark.cpu_only +@pytest.fixture +def forward_case(): + backend = object.__new__(GlmKpoolSparseAttention) + backend.num_heads = 2 + backend.head_dim = backend.kv_lora_rank = 4 + state = SimpleNamespace( + latent_pool=torch.arange(64, dtype=torch.bfloat16).view(4, 4, 4), + block_tables=torch.tensor([[3, 0], [2, 1], [0, 2]]), + num_contexts=1, + tokens_per_block=4, + ) + backend._cache_state = Mock(return_value=state) + backend._dispatch_sparse_core = Mock() + return SimpleNamespace( + backend=backend, + state=state, + q=torch.ones(2, 8, dtype=torch.bfloat16), + indices=torch.tensor([[0, -1], [1, 2]], dtype=torch.int32), + metadata=object(), + ) + + +@pytest.mark.parametrize( + "route", ["context", "generation", "verify", "paged_context", "paged_generation"] +) +@pytest.mark.parametrize("supply_output", [False, True]) +def test_forward_dispatches_latent_rows_and_preserves_output(forward_case, route, supply_output): + case = forward_case + context = route in ("context", "paged_context") + q = case.q.repeat_interleave(2, dim=0) if route == "verify" else case.q + indices = case.indices.repeat_interleave(2, dim=0) if route == "verify" else case.indices + core_output = torch.arange(q.shape[0] * 8, dtype=q.dtype).view(q.shape[0], 2, 4) + case.backend._dispatch_sparse_core.return_value = core_output + latent_prefix = torch.zeros(3, 4, dtype=q.dtype) if route == "context" else None + selection = ( + GlmKpoolBackendForwardArgs(topk_rows=indices) + if route.startswith("paged_") + else GlmKpoolBackendForwardArgs(topk_indices=indices) + ) + output = torch.empty(q.shape[0], 8, dtype=q.dtype) if supply_output else None + args = AttentionForwardArgs( + attention_input_type=( + AttentionInputType.context_only if context else AttentionInputType.generation_only + ), + sparse_backend_args=selection, + output=output, + ) + actual = case.backend.forward(q, latent_prefix, None, case.metadata, args) + torch.testing.assert_close(actual, core_output.flatten(1)) + if supply_output: + assert actual is output + case.backend._cache_state.assert_called_once_with(case.metadata) + case.backend._dispatch_sparse_core.assert_called_once() + dispatched_q, dispatched_k, dispatched_indices = ( + case.backend._dispatch_sparse_core.call_args.args + ) + torch.testing.assert_close(dispatched_q, q.view(-1, 2, 4)) + if route == "context": + torch.testing.assert_close(dispatched_k, latent_prefix.view(3, 1, 4)) + torch.testing.assert_close(dispatched_indices, indices) + else: + torch.testing.assert_close(dispatched_k, case.state.latent_pool.view(-1, 1, 4)) + expected = ( + indices + if route.startswith("paged_") + else torch.tensor([[8, -1], [1, 2]], dtype=torch.int32) + ) + if route == "verify": + expected = expected.repeat_interleave(2, dim=0) + torch.testing.assert_close(dispatched_indices, expected) + + +@pytest.mark.parametrize( + "invalid", + ["selection", "v", "out_scale", "out_scale_sf", "output_sf", "shape", "dtype", "device"], +) +def test_forward_rejects_invalid_arguments_before_cache_access(forward_case, invalid): + case = forward_case + args = AttentionForwardArgs( + attention_input_type=AttentionInputType.context_only, + sparse_backend_args=GlmKpoolBackendForwardArgs(topk_indices=case.indices), + ) + v = None + error, message = ValueError, "quantized attention output" + if invalid == "selection": + args.sparse_backend_args = None + error, message = NotImplementedError, "pool-expanded selection" + elif invalid == "v": + v = torch.zeros(1) + message = "v must be None" + elif invalid in ("out_scale", "out_scale_sf", "output_sf"): + setattr(args, invalid, torch.tensor(1.0)) + else: + args.output = torch.empty( + 2, + 9 if invalid == "shape" else 8, + dtype=torch.float32 if invalid == "dtype" else case.q.dtype, + device="meta" if invalid == "device" else "cpu", + ) + message = "forward_args.output must be" + with pytest.raises(error, match=message): + case.backend.forward(case.q, None, v, case.metadata, args) + case.backend._cache_state.assert_not_called() + case.backend._dispatch_sparse_core.assert_not_called() + + +@pytest.mark.parametrize("paged", [False, True]) +def test_forward_rejects_mixed_phase(forward_case, paged): + case = forward_case + selection = ( + GlmKpoolBackendForwardArgs(topk_rows=case.indices) + if paged + else GlmKpoolBackendForwardArgs(topk_indices=case.indices) + ) + args = AttentionForwardArgs( + attention_input_type=AttentionInputType.mixed, sparse_backend_args=selection + ) + with pytest.raises(ValueError, match="phase-explicit"): + case.backend.forward(case.q, None, None, case.metadata, args) + case.backend._dispatch_sparse_core.assert_not_called() + + +@pytest.mark.parametrize("context", [False, True]) +def test_forward_requires_phase_specific_latent_source(forward_case, context): + case = forward_case + args = AttentionForwardArgs( + attention_input_type=( + AttentionInputType.context_only if context else AttentionInputType.generation_only + ), + sparse_backend_args=GlmKpoolBackendForwardArgs(topk_indices=case.indices), + ) + k = None if context else torch.zeros(3, 4, dtype=case.q.dtype) + message = "contiguous latent prefix as k" if context else "k must be None" + with pytest.raises(ValueError, match=message): + case.backend.forward(case.q, k, None, case.metadata, args) + case.backend._dispatch_sparse_core.assert_not_called() + + def test_cache_state_prefers_live_metadata_and_rejects_graph_fallback(): backend = object.__new__(GlmKpoolSparseAttention) backend.layer_idx = 0 diff --git a/tests/unittest/_torch/modeling/test_glm5_next_contracts.py b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py index 23bd74dcfd2c..c71fad89dd76 100644 --- a/tests/unittest/_torch/modeling/test_glm5_next_contracts.py +++ b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py @@ -141,6 +141,57 @@ def test_cache_manager_routing_guards(route, monkeypatch): get_kv_cache_manager_cls(config, kv, True, transceiver) +@pytest.mark.cpu_only +@pytest.mark.parametrize("fp8_kv_cache", [False, True], ids=["bf16-kv", "fp8-kv"]) +def test_fp8_kv_cache_rejected_before_manager_construction(fp8_kv_cache): + from tensorrt_llm._torch.attention.backends.sparse.glm_kpool import Glm5NextCacheManager + from tensorrt_llm._torch.pyexecutor._util import _create_kv_cache_manager + from tensorrt_llm.bindings import DataType + from tensorrt_llm.llmapi import KvCacheConfig + from tensorrt_llm.models.modeling_utils import QuantConfig + from tensorrt_llm.quantization import QuantAlgo + + config = ModelConfig( + pretrained_config=_config().text_config, + quant_config=QuantConfig( + quant_algo=QuantAlgo.FP8_BLOCK_SCALES, + kv_cache_quant_algo=QuantAlgo.FP8 if fp8_kv_cache else None, + ), + ) + + class AllocationReached(Exception): + pass + + with patch.object(Glm5NextCacheManager, "__new__", side_effect=AllocationReached) as allocate: + expected = ValueError if fp8_kv_cache else AllocationReached + message = "glm5_next does not support FP8 KV cache" if fp8_kv_cache else None + with pytest.raises(expected, match=message): + _create_kv_cache_manager( + model_engine=None, + kv_cache_manager_cls=Glm5NextCacheManager, + model_config=config, + mapping=Mapping(), + kv_cache_config=KvCacheConfig( + use_kv_cache_manager_v2=True, enable_block_reuse=False + ), + tokens_per_block=32, + max_seq_len=128, + max_batch_size=1, + spec_config=None, + sparse_attention_config=None, + max_num_tokens=64, + max_beam_width=1, + kv_connector_manager=None, + dtype=torch.bfloat16, + is_draft=False, + ) + if fp8_kv_cache: + allocate.assert_not_called() + else: + allocate.assert_called_once() + assert allocate.call_args.kwargs["dtype"] == DataType.BF16 + + @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") @pytest.mark.parametrize("deferred", [False, True]) def test_encoder_only_factory_materializes_attention_weights(deferred): From 90334270bb22559e4e397ba08033ece6dbc35604 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Tue, 15 Sep 2026 07:54:19 -0700 Subject: [PATCH 03/35] [None][feat] Enable GLM-5.3-Flash attention DP with MTP and FP8 KV cache Keep mixed context/verify batches in one MoE call per layer, support FP8 latent KV storage, and guard KDA state offsets against overflow. Add focused regression coverage and update deployment guidance with the four-configuration performance curve. Signed-off-by: Ruocheng Jia --- ...yment-guide-for-glm-5.3-flash-on-trtllm.md | 21 ++- docs/source/media/glm_5_3_flash_fp8_perf.png | Bin 145567 -> 169957 bytes docs/source/models/supported-models.md | 2 +- .../backends/sparse/glm_kpool/backend.py | 38 +++- .../backends/sparse/glm_kpool/kernels.py | 61 +++++++ .../blackwell/kimi_k3_kda/kda_mtp_decode.py | 4 + .../_torch/models/modeling_glm5_next.py | 68 +++---- tensorrt_llm/_torch/pyexecutor/_util.py | 4 - .../defs/accuracy/references/gsm8k.yaml | 8 + .../defs/accuracy/test_glm53_flash.py | 33 ++++ .../sparse/glm_kpool/test_kernels.py | 94 ++++++++++ .../modeling/test_glm5_next_contracts.py | 171 +++++++++++++++--- .../test_kimi_kda_fused_verify_parity.py | 45 ++++- 13 files changed, 449 insertions(+), 100 deletions(-) diff --git a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md index 2c1e2c63aeb1..b6cb19f915e4 100644 --- a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md @@ -14,7 +14,8 @@ The following features have been tested on B200 with four GPUs per worker. Disag * Overlap scheduler * Chunked prefill * Multi-Token Prediction (MTP) -* Attention data parallelism (see [`enable_attention_dp`](#enable_attention_dp)) +* Attention data parallelism, including MTP (see [`enable_attention_dp`](#enable_attention_dp)) +* FP8 KV cache * Disaggregated serving * Disaggregated serving with MTP * KV cache block reuse (with periodic Mamba state snapshots, see [`kv_cache_config`](#kv_cache_config)) @@ -22,8 +23,6 @@ The following features have been tested on B200 with four GPUs per worker. Disag ### Limitations -* Attention data parallelism cannot currently be combined with MTP. Support for this combination is planned for a future update. Use tensor parallelism when enabling MTP. -* FP8 KV cache is not supported. Leave `kv_cache_config.dtype` at `auto` to use BF16. * KV cache block reuse requires a periodic Mamba snapshot policy; see [`kv_cache_config`](#kv_cache_config). See the [Model-Feature Support Matrix](../models/supported-models.md#model-feature-support-matrix-key-models) for the current support status. @@ -131,7 +130,7 @@ EOF #### B200 FP8 Config with Attention Data Parallelism -Attention data parallelism distributes requests across ranks while keeping the routed experts expert-parallel. Set `enable_attention_dp: true` and reduce `--max_num_tokens` to `4096` in the launch command below. This configuration cannot be combined with MTP. +Attention data parallelism distributes requests across ranks while keeping the routed experts expert-parallel. Set `enable_attention_dp: true` and reduce `--max_num_tokens` to `4096` in the launch command below. To enable MTP as well, add the `speculative_config` section from the MTP example above. ```bash cat > /tmp/config.yml <DDuow+lYnUfrh-(A*ct@nM}yB^A3mE5*@-)1^Gx@}Uw z|0YjINB<)o-4C~a+=M?FdA2tV{}Qp2P_>n87r_I6E6PlKSWpFIMODQzwf14IZb|dX=$(C-rkvB zZ@%F`4U1Y@lTP~IfB(L)uyFMFaan4Tl7g983d^nT@2bPjsGL51ns;w*LfzT5I}th_ zE7bnUSGMvB3iwFIG|B-X6aU<|Tok`Fjp*70>40O22Gt?M&6&|GE~6)SG;%KEHOpp0 zBO=mQmgY5x5VQRyw-d z-*aHo`|mAuoDkl&m;c@8zCK+)y}4_T_X%B2FP;slc2A}36?46nc$1sgXTL?>i+8gV zMblR^-9Ch!$yK!0G9PWu+#dG!P6XbfKyNvaM?*x=B22{oa#awYV@2~u>FMPy?(Pfy zyS-P1&6=-M#`~*+wD#o|`rn_wc;Nyz&RM6dw)c34yy~XelBIFmw!CrW){%THP~3qt z4$Y&EukUQ!KG#y{XjwnmS3c&ol$zVgzj4#1z-Gfz_m#^JA3oF`sb@(mdC)-%5vk8j zE&YSzX@#KMqFv$ADVN!lw*!5lmL(^!M#=9=l2y_zLd4t(-md6Wt9RwBy8RUppg-J} zn@$zRdkMzI$A^9l7d1_ieQ~F3@`UdDX2xso?j?!Z#m-D)ZMl{cGqx**N4HEEqa3xq z--_ySo4@zzHL=<&Cp%*DZ3m9=@zuP@k&=>H{1R%}zAJdeVYKH z@D0lvU1n+}X>YdhaS8}zzY`1MR*U)=c4p+=!(Uh?rq%m=ni@i}`-I$=Z*@3!xrsPU z={=-fix5kFa#%9ZIWQ_ZD=SOT=+i+St%8PL)q#&;6ukCEdirD>K=Sn^4!9Urt}Rd3 zh&azANG!O`b*L1)LbbjA3n`Je`MU#Dv}^l(#mBSQ-xdwkUlXS|nN|3)OaK1+*Xx2! zfkj1UMOS9(JGXCW%D`b2o%;K5=l1B!A|9(n+D`q9L9@Ys{S}GrT=``(Q6XMrIrh)H z^b98N{@lSIe3@lrTD>2iUxWNjH+;*^?QfH0f_RO)U;Uw;dGmf*W|~%^$xvPF=FOXr zaTm^Yy6_Cg#>JVmWa+c}=#N!xTU}i(7)dK+euR=k$G%?0yY;gTYZ{Z&OH630s;bYl zT_+{*@vezrJMik1q}UF89P$-k{q!MNfcEt*1J~KJ?QeJNw(Kl&60!aAh^iamI`z;q zdO_yat-Q+|3J=9myHaG{Cu_=W=!+A#N%At%l;Aj}jbw-}w;I=pvzc^hZ9uORyI5T5s-dUtpBRco| zm`d7c!?W4S4<8iQR#zw{)MMVd9fgife(W+I{Oud!8iA}F!f_IIe&g4N!FBICUEc%dX`W2Ox$J88i z4e!|qx7lFHMeY&lrI3QEj2mxXI!N%NnPu=qRDmsHu^&=`v>_ zI@_%4_3)uwnHM7?GjmFFvug3ucp=W)rv$lZz2keH?<+jDckkZert}E(AzkYf!MGF+b{ef>MSpBT5}UQDbiQ2P&f^ph}0=l zx47Swl>@hKx?3IL#mIH+*IzHBYUDN+&vhoZWE+mu#mcbH7NW?A&V0IP(osNp^Y(53 z#6&t45)Jd3dF#vC1CH%4nzUW6h_B8^Yq?}=nWMY!J8|>gFg5uIni|;`qAs%yC%asA zCwtt~Y-x2-j9khMFOBQ<1PfM6UFW(yO3O_5{qjpRPKtLyxi6~<&bq|PPrq>FolQ#? zGH-1#$)_}Bp$smfvMI%1JrKGA(9x_}Lc=~~(zpHcA(PXzDJ?8S$kagD;P{@DkMm1F z5B8t`-jO@!#AE$gs91R!kc3Gmd~$la9@~w_;0q%o6H}aVeO$gd;~(C;`I7MR>bVwS zWtnIaL!PYevUp_XIk`C>jIcZZ$5NwDFF%l1Q1aD4)j5^OGiBCB?rU zKCF~^Q$i_8QL0JHG0d4Z9E&z;f}^I~=TPnvl5HLEu4V4!^0F(B`pe%LL}&iO!+P=O zpFa|GbM~wz{`3>da$0rh$urEmfweAP#v}e3V$o3$%%--SqM2V8+o!r;cm8V+@5l&_ z_enI)kN_Udyo`+MGVk424jnpVP+qz+tC8#mc+|vRmO*`H?YRDDJfFmuD!Wqkh&0tlP|U)y}SM+O{t*#Y3fJ9G&+iKt{5gw%a&P zf*wWNTAX6qB%G>U+&Y|>n3x!cVlL9q_3VT$v)7u=+G0?Jkzng+y2sioZ~i5(PI~N0 zhl0}Y=h|(3)rweMzy497#~+NkW;m%iUN|8Z{~nb{P{^{gJ*tbI)%8<~wzhV7BDw_= zZE&kJ{{`S&xnLGGRXxlX8;v7T38BM`hjlq4D&A>Q zTDrDcNOdn=8b6aetG16oh=q`XMaxJY>*37KVwX^|1S(zR$x1AZcNA)*h6V;2 zZ2s}bjD)q%Pp^rrFy~r!Srqg<*gxvNmbbW&zbIO*WSu`PYL3^9OGw}>>@8TCZM!m1 z6T!E#1yH)^!CpdbwEY4CVk10OTcV1vV+#5MHHvc@l>mGp3` zI~t!U#TAv(Vr*Ky=VMbto3MGSo}2?75kcbkhz;SQF8)uRyui*F`uoY@5K)&LY@&Gl zX>0pLlN!!#Gf;v`=U6VaDjU773R;^L}`YH#V6 zz(S?xn75gf`y9AR>r3eEtZzRMls#cwAAe0H*P_F2@YA8f_o3n*CFu8t_8dYuhi&Ub z^OlsnydRE=nVdV#CalTJy6=6k=qfgpefO*Hq`p*KkaGU|njUp=BC~n)33?7Ti7DVd z??mDCz5L)HpFYU~3)bE|O4%o5VpJO?QPAgeMpMjPPQdUZw?m@f-PhOcCpabdY~3Uw zDQViCXH9B30i9R_&NF8;KzD!zV@PAd_YI{~ef-!c!&j`Q9EgpW$)-gcrc$hWF2u&f za2`1#8GkiQUQ_cqi^uY$JWno=*2GYwh9wn8)*l-VdxVppzb>k?oYliD=dy)bcSWp( zcfokU$V`8bj=I12eD_`H%ajSAN?2vl zL?V|75qsbH+xQvfD|;%f`O+gZg@Mn}y+RHVCHryp0dn`teCf2qtfv;HD# z-j-wT-2DeVx1YD$;!G_toqt$ZDw@BV>l1U-<|fxH7P{Lu6;%RzfRQUxK@C;4l7;{l zGStNn9eVLjKOC#Vn{1v+weQ~<4J;}g3?PZ^Q(cKRWl8#lTClIL@6cdvwD6Km)6VGG z)q4*fgviUwYwo=Ghu?d%wwzSst`axi1NGt72a0`0wQe?Oi?7T)mC*U)k3Ztt+uPrW z;A>?iOxMZqwTud2r_Nwog=)Jm=oj{t;@vN|w6qx5_x1LsG>TevpGyok{Z56i#>mvT z_WNOi;UQd}kyCWD)}A{h}J@!=7;o;Iha!zHxGwOT)Smr=HTRmQL zST;o^+wc*3z5o{s*2uYL);O z0|P@-mVVW)efvxtb#hk6+e8d%mi{=iSj2Q{IwX+2F_7xdhObh*wz@p$S*P&qD9|n6p1ou|JKDk|(CF&w`aI-`!{Ur_syd}I zieRAJ(}!z-4_V#kUsS94rQNz_E2*i%HRhs@3fGvXWhyrS2pf=|p01{L?%Zt_w;4I_ zf<}YD4Rp7&yopqIJ4i8Ex!Pm_Jyn$X;Yd4{-k?e~Ox!~o1m{GufWxS&@4KB)h5)UO ziiv6SmCQ$x0}PVNB`By!uw{INd(y@&OseMlckdo}vd0_Uz6l#iwB%&;na}itC<&BR8+LHO{eHq=GQs&0Ip-lDo5vQ#Me@w5mC(Bln+b#G;pOC zMd5>SiHVgcPIkIZS2OA3kt&b0E*|fy$68g3J*PF)MX5~B%{2i!nuqgAnKUFkDOnum zVz{J#6ulJnxPd3Pz4w!8o_-Y*#j;DAo`FS>%lFSeKLK4f#7O;;JZ2WgV5&p+R~p+C zq!}+#pT7}Wu6y%lL$={H z-*;0r9;*t`mmVJzzEPKJsY4J>lq)JB@8dE(en7ONe0qH?Hk8=xvW$GUu{EfZ@r2uEo45Cnq>6I=XnL%-aXsXl1)~R7C;e zXHe?ZiW{~1hfNok7p7Z_oU#E=_2e9@0(nBP^|e-bwTn~%I*#P9s&7XsY8 zr?7)V-EJF?-L1>I{^8jPVevnqBANwjbbf60v>6&2VyX(uzf)_Z2u-%3z}~cYrcOq^ z#5JGljHltS&>U`qGo38+GRLArJy|*R*k$!+mpNAOJRE3FBdNJ2KvqOc*|lens#%9_ zxz7+fyD}h7%E$1mxSM5OO#1p`BE_xLrU8%`bJ_+?(~2+TF?`KS4L;<~c!SxcGGK5&qxvD~h-4<9aPPXV?K2I;sTJ+ZefLGEyJ zb{v#q6G}iYRw-Ow7z?w0#+KmSM$&2jOj6F7KQjtbO}} ztq080bR7b&fb1z7GAO5Ms^xY#gkC&R=zRu=E%oJuaT$-w=%4=nyMkt5EVTQs*QMPp zA=|$CdMsZI;?tS65=`5H8ua=uy@o&i#*G0cmtIjH%4vwlhB4aCf4$+I_19lN`*Ks8 zOWa(N1(L5mKXqfaxntl}0I7iXe2I_@5?oiP)7MxP?-ejGn)+NJ>|08nSE+k3{o#T)pk2eJRCv#bkx(*(M8+B0o2Zr3=M6U8yR7ZjjvH zXkZJ0sdjkONJA#@Of(+r`j%jqbp%iZ0%%@2S>!S|P9zkGg-3e@1YKs$yr<4g6xEyf zRqYTmX~<9?A2gFuDUHuR#_O2D7#eM3YpYeqKu2dKzHYT>&p<`fSjlk05i5X9!DEoF z$(1rDa0;T6@0gpA{jg&3e6P1~P$yT0-H^O_dtPl+d*dUYHjBdQn-3cEtoyi!iDsyt zXLaLY*ZVFUSrqQ;a_&46I~=}z*z27oVn3`}{88qqAVf|Dv}${x#t~5UCNx+(JecJ4 zwwpX=nFt>i1rv=~NIl$V#kI5l3f zWC3DZmhu4=^=tQQayX!6ljZI3SX)^{DdBDvhgilkG_5Wh(ln#K1M*DactKUva(0ba zJw$@Yckh0~DqTexs-K_i;~6e{_b$)Ar5Vy^i-DgN+LptSpHg`QOw42N;aMju#9xiw zlJh5ID2evK8jgZ8bSjrxX$y958B=@<3%VxZpLR1ORcO@?X{*gV^usq4Hv zSxzfmTdC3N0XJ|wj(@xw3HMladde2>vCaxs&eRVG)SOkgsRLrbOP$VXB$z^msv-xy z<9uSoI^Yft4pv?|AZYZL#7d@axq1T;)Dp|nA(9Tw4Aoza*zmMsap3^5WtQ49Zrr7c z0X1u?9uX7kGznW>Sqk0GeCm2-WhFma`bS7+3ZVntz^n#+n>KC?W(&Tlt9xalbgg6v zY+EQy)TPmGVM<>CKH5=U-j9mu+M?d>et_}P}k zlF?e27~TU`P+;?8CZH9l&U9qzN-9`bq#Z>ERRo*dsq>3ue(%I*Kyi!O2^@>(!_yIK z%kk7(@t1?;lUB*{0maR!M)+9Kghtbv8kD;}E^iYIHd{N8IX6I5IyVjuju3Q68EO}f zP?)pWYBmK-y0hGwsNv}yC^jPnrX?3`UKL*X>S3%x#_S8LQz#aEgFA)}qA zuL9s(Sxm~W#`tYW+# z_-ASZF|Lj4-nG35ItpRVd39lchFxNcO#>%^KzY(xN$a>2EeC~K&d%;71g`kPi4u*I zowuT^-upg$So{36?ab=Zgeiz4ku~6Z$D*vk2)%ax{COfD;|n*$UD;2Y=+ZgaVfVD^ z{ag;5b#k%@8aMdx>;)JNCl{9tfQBq25+N`_@Tbl%?lLOgzP?R>`xHKHq>-pILFFcn7BV-X^2+=RFHV#{${U7j})QR*uh^j0!)F-eB5+m7?rK8|0?=l zBdf>qivTVa6R3!-1@_5Mwc^lvVLEbxUxRUOW>`;Wppxzfi@bzgLQ{Q&y5_X$X@%aD z!+MHFmPHXNv$3(UU)l$0Cuo<&%saaGq}>2`>7Mw5{Sx-!x1axK0y=-bhWcN00Lu5j zKQ-4k9s2kG|9>d=Gf8t;f5lZXS2vzD?-Q=`R&MYnh1)bsT-C&vzuslyQI7$bx&mKc z`r0)WRAVdxJK^a#DhjA*c72aeCSCO0o}`=_BBN^+SfN)3U3~@{XehVKozy}Kii%dA z)$6G7q56167M5|@HMGY%I7$?=7Jl?`RnP_{E2|6`LeyB9-2}g38*!g_kC$b9pNEHz zUoXdBi%#c0q8;3%_&;e;u<+-)@obRj)Tb;>Waf~ zzvy=D`Kj7nyLKtwEM1ou+1Xo04Y8-UM+{(h2VV_8tCnQTZ}a*0(3MWqzu4hyHooE+ z`JO!}5w-=TpqK9(zzz#*R3JcHChfCh#dam=(FiL(ixorQo{ji-HzSsy4hl>Zk_95| z!U$q)+Yu06p=TpgO5ArW(rJA?J%3e4Z2#f9SZ|{8({35x)XdU*DB?Jt5);Gb*;-as zW(*%`yYD~>mXUE)%;;03{M=Jm;_&lFpk*t#x)ybe_4f4<-a>- zpZgXfd({WRA>=$WxbrDze+xiX>NT;2cQ1rJRu|$gJ=)V#cWV0qkr=dy*70?%5q6m7 zzPq9%Cxw&BB*trg=QuizV~2eZ5=ctJs`zItK&=dHKV{$l`SN_Z6zg$qq6#xPo718|j0M;B^l3J3c> zVY37j3OR5Iky~FQ$BXB*y+z=LQL$CMA5&1d7Ll`hX;{O8sxt)jjTjEqwr3$O;yqg# zmym`r7WS^Hsi{eG_;^gjB?AkYq*Lmixab6>gmL$7Lph7%djuh$y&0JXz#|?0(&K&X za5Ch?iFZ^q`G9&NZ`He;27*a<>$aoHDe{27I{tcQPTXaT(!@PXED`^R2pzLEQd@^U zo|)jbb;(f6MmLb3cr9#P$L8&@>F(>7I5pYMSh6?(kcub^G)A69^1O!t9t7Fh*qWJ7 z3JBB_1xMqt%feJY`}>u*4C1QqYSB)7M_)G^%^bIw0!Ykdi{OAZMLyG6hDCDW@c?du^FG z;f(vIsWM!fe)z$t^`$ZYzK*r7kJAG+BiK#HTwUSE1|&sX6JKc<&Wd0Vo4=kZqvEsY z#09{M9L}1*|Ndkc;btKBk*0Gt%>y6t)yjO(G4fe`lT|aLB#;!S@(&FBa^I*j5S}wI zRdrMm{;53DcB~6>QsVn`bad*JI>m&sZG1SEvFldfHZ5$^Sg+S|fYM69{%nbO42Tse zB`&Jvm<3a(1|mF4-sR*BbL*6}M?D(p68`qi0^g#TUr(93xE(lqOBVK{GsHn6c(ZeG zI3MDlIAb^X7qmJfISbP-3g~PpffI-xDE4W{r2MdPBV!?bClvEYJdWLB7Vy%;ore?U zAM5Osm}6xYG?GnR{2O1F;Lu7UVhtLEP-+S$9`(xkro zk6*w2=FJ=K${I9mEZ;epOe%1-~L-dn{t)_?)u=`h95m1gg4`ldUbrvQj1AAj> z(<;@#&%o!4Q&oICl@oBQp?|5G_l?Ce+S=HBFc=CDKL5<$KL%yjdn!4I!c7<{Pvz9q zlpU~0Hh-E&+qWW8)2)+2p2^uX`B-Uk+@zO(UcvcWgyt{wMeE?xJ`8O-7f~&9cPPQ_5#0csZoQ&NCl3N zr#9;d~!oR7d$bzJ4=D!n$qg4P? zG9V;m_>Uh&YZ~Pr;7^pRu5dhsp3Yqf3UssS7Gw?;9CR9XT}S}ot>iOP^#x+HvR~e; z)l)NuBxA;R!)d(zbm3BlQZ9C?a#j8+&p#jFb*Uz)XaQlKhiO|iD%dn+uE)j2sfY3CbM+`Spz@L??kze65g{H%y&A#G z)j4?H(~EUp#A>Q|dW2ny@tC44YzT<4o6^UT}Rf&;Y9?&AbktBS*?Wr9~<;G z|633|^pj$eX9W#cgEvYAuGLAlcN}jY>KQzH_}DRNAaACANgw7+^)>8A$0o~~|gJnn%o`D%f+OL@+A5>8Tn^lELt z1iVHQ+u2~zJ9qEmWOI2>iLWgy{PfdL>^b7D^BF{Jkn1FMEngcsu@ECR^5E z@8e6NtMh%tFe%G~7{&M7XLm6%F%k-rSOC*aX%>VQe=H5coZwfo=D z0nfh;5?!6-x?Td&>>SZ9u#*DbqHw~Vv63{>e}fXHFP8(SZ>*!RXsfEZv=DLpnv>9}z{uZHRM8 z#7AgfT!<8q%*=bA1IaHgdL9!O?~3YBxF#A4Jwesqc4fEkw`CW}E9L zC^I0XkQ2HO)LLSsp3Xox40L{u5Yqt>yWe3{WO3o|9E{HVJRvDI{XxJP%#GQ{5I!=& zX==e0dRZZYQwr92L3wDf0Xx&SHn(e9vNTYSFiY#g*}nlY9bS)0V7Q6edI3 z+v#5Pw~IvJ{Oi!g_FZZ%H{gTe`3}Q8%P!#Sq@tR{Mz}9ZLG+@Xv+E>Hw*Si)X+Kh* zu&WAQu7QG@B51uhnq9-JUhonFa{cx9D84NQqG@Y>|Dc;Pl}7URw=_wag^a=p zeU}7a!6Ar&|1X(=xeUt@G#NtR0T^r-NO%$&B;&4KpU1w6zQ2Bauxo8Kmpm3ix6}*l z5=kZl6lCMZjYs+U&*@_ozsAO+YriH$Nm|l7;TR%V09uTSh!5cpx$V zh`fGq#FOCccEH3P?kp~Z^+!mLfAh(vQn!CF`VEG4c*d)6JUN*&y4I*^)o}dLYweHk zhCk$mt6F?|qJJ>;=vaP5O-*jI_s8Jr=Lmmbw<0JYWWK4%{doPzM`S)$p_b44(JSLjbSu4(yKjGYH@NFg~ zZLdU-+05kRlh@b3+WbcTlhxS&Ig`SDf(!wD`}u!)LZzg0J6#j+-Phe?|1(XqDvW_0 z$@e$@4@Q3c?$1}J6@~ZD7fM~Z62JzO`lP65&gppdbpK`(DXv**D(*wiN=%EDX0mnr`fmMKRDvv5o zKrAV!ga$yPpw5y&10r7BQu|r*&)#L3+wq*@w&dUj-D`VpvRw27B;-V;>i*nunA>M*X^9IN zXs9eClZ`0)*svp77)c(pOA+`-l0TlhT%^8-AS!stmn9V`T{Y&?MV72t&6FW1h-Hm> z`U_{fTYH9{%itlX_x*my3L9e>*$y^i;I8|nZKkA?3Yi8%*oj8|z}syzA0jT`V?!t_ z#3dHI@#cr19LG&RU3hhR`6A1I2kvzHMRMTi2CP|Mt8LlM+kpLdz+yKeW9r;!|tj{b{;go6GSOLFT}k)=35P+qA$^+$O+?ka5Sx_@eaOM5AQBpxuJIAw!G*`^ z9~2aS{_dvJ?TxLr)^VvPYc|L}&ziQsZ~6&#Ha%51V>b*83CANTL=LbzX&oZ2|es+HF-sjJy ztKFq|S|Sdk&xl+H^qB-3f$*}D`Q9z;?CgmUnaKPgaRT261MYGzi?Xr3`*KRcm(je> zY{{>=@QclPzg@R=ne$n{Q|<+~aKLW!UshI2=MadnB(0Gs6##YQMb4 z_;UnjA-cl|G4b;8$!YOTb@{u|Hd9hfgyEc2yrcvz3k@&B305r|a@q?MjuLeh`*TS%lCMqJZs_f01Q4omPrC^ubr)@6Tas5iIE)1mmrDxho z*f^dhHur55*!uR(o6+%KZnLA;|5tjf{wb*CZi>z;M_g0iDGnIi3@`pmF3I7Vd-jDL z0e7X-RPX$qQR(Gk?O(sLOX=(0ENE;?i%E)!erg?1q{t8q301KXjAc*`>L2bltAngMe_^Q}SF5ecM zjx&9y0^i^*=@6Co9}Aw3P%=Ai^uBvYcuovgmDzY_u?85jUkt~U5FgL`K@=6p2tR#xOVdA1HLeiP#l3+3d=T((IZn(~{^WN12g?JwjZ*0J_Hy=ub zcI?=3ztqscfN8Z874}UR1M{g!M6}h-J-&9Imr^m{ZUl|f|F4=_1yW){5H%rAB1xE< z2^IB~`;Oe-tu2ePM*_n^?=x`_afF6)+AI`M(PXIQm~@B-7i~U^p*Y*n^2$mly3zA@ zKK+ELDQGybOp`a4ZoFO%S$vBJU%`7Tm&Bmek^&2SfwYqyCpu$IH2z%+l1_Vx zDKrpu6(aRO$m1Dyh1-rU-(84FiVr>=P6J`Ye<$39#Je%&C3vO;S}?JK{RH*?{@J_a ze@bt^oSv$FJSIgCT~rIQMMso<-K6jGbKu5V4l1$8Fkp9#jqNv#(A6Wq@WBWBPRs8l z?3~9?RsCiVU&McYmv)MYPn#Qz5FY)R`K_AKDbML&v9SFZCJ0}n7~UniOcNx?UAuS3 zgNsQOWOYs=U>t=w>+gPKx_AKtHxyQJ_ad={&%Yq8B?sS!8YSVqc~1DO`*I$M!2rF0 zpi4q6&)U-kD46E9cXt+}VoT{~uWuWYbJtuiXZ(^X`o;r~Q{_DlI**GLL^Up*5nZJq z*R10QX*Uibfz}j9gw5Xc!9@<&dYFl4hDUmh%)hqHCUe^i;89E5k*&MPAR0sDW-&3i zIBdqKXC#xlr3zS8Kl4$1L`1}w55dvEFvd0EqGVPLBGIm$JEPE(Mi934i$TCfjZ6@i zHN)H_({Sh<#}WRL1E#5mznuall>CZr2*rd<^D+MXbN3X!nYl~u^G~_IoE+{P5J^xo zGq?2o1`%b`>`KL?PY5Lsze?0AY@^pOClInlYv^y9U& zllKVQ18q+Hs){r>$=?lO4mLA5a=(aO0x~IN+?FgINm8C=+P^;;dVnGTPv+iIpc(%w zT`MBm*J9_JINHBtG_~JUkuv9-q`LNO_+Cfki!reo)#u-yUw?paZ9;na zHo3K*bhv1}u{%dR>FI|<%RMX0jb_f->0gtTmF@8hZp3iF6_RfTUh_&de9Se=I!wriS+u0K`vkq#TRqZ#eM;%R zPWg!Zpb4MEI=!Zo6%+DGNus6u*0y$d3Yw2On4i9-xjQ?Hk-0m)c~GRFRKwK1P%Nqa zBePvD@s{ufdK1+kLATp^t1B+Ny@D5*3W;Jz|`$X_V@WB&1oy>w~c`LNNm zALTo#qjRM>vRLD4-f8~zuR!<81>rwDkO5QvoK;xxLD*|LgD2-A=}}4!8M->z))`WvAcr7u&z~{4Nq-tc-lL zrA1neT0Qjn`X5E@u5M?nqxpID3S)?^Kz?3P$UK1m`SSrQjP6zH!CUirhib~35?mri zA|oqTd58W3#bdu>szYC~Y_Z%fzr*wDk)PaLA3JxKT{U6SXp?x-QiIqz!bc>J0n%Gu z&S$keZ#~vks)Me(ol>FNLZxk4YMU}$hfjx8JvO1p>|N7DNaWuMi!IF($fS;c< z9Uj$;$P8|DE_@M;j4NP?@KVG`gdMI5ao9nInGACf*X&I>;2X(&rWKBhUIiRa}z3v>;(Av$)B=->jP6gw!NK8 zs*`jDxJ*+t`|4G{el@ANpKtg6Z#wO}v@Qiz?z=XiNn&&QieVZ1cWFg?N158mH9S7# zHJGi=|D}c6xWD2xWCI$|7y>VUVPp!!Y1SmLPG%H|iT?Ss9%=G0S@Pj9ydalQ@B{*i ztdDMkizUOAY7te&ObVIZhb*E1jR3hrKhTvZxDhb-WDshUmX@wVe@@Bqz@#(2PZZ=k zPT0}J2Z!uH=179Hou&LBjH8%I(AhcihU%vWsD=%K)mGW_tC zJ6tA;{bIoIaZ^Os=q*{ZSE<#3SBeN%}nHA@UDldy(W2e_j z^jKTHU#ih2SqlZqZx^Pe(m3FeE)psi5nZ(=N6jr`mjl%^^l20Zl6+U?Bo^)F@SQy zA3&HDaoEY0T3a3v7qRNTja?E&E><`^`Te&Ka$5*8Z}&bbcTUc@)A*+9Tb9|E-HrJT zl=;p&h@DoTcMNN^KY7WSEf8T6DS6Om=?+PJs1%M4>D~{|e3~IcbYkpPDl7PJv(}f1 zWcsXlw$(I5(AW=+TJcs2@p}=arvYhHGzI1A{PX=g-yZSD_b*J;wH^B~FI$=}&fAyf zQte~DFqu)l^WH`oOXU>R%m%l!aLe=^bl$+}GeH6j`QJKZv`t_T;S9OW_jrb!wvA&W zx8k5EkT6|ceLW2tR-$sMI>C0LdqmgY@pdeH4XUu zJX`~>!@R^MaL3BJ4NzS`u0Da3v4xeKz(9keW<-)QW~H()ULv`vUdN-+anSd z<+*WRlaZE|pBwM6N5|-}E4MneoSnwo{OQxvR zGp_#uPvLW7{I_Rwz2~2p&D${~EH``PQdcqW)QSlPvazl_KP887tYW5HI2<`#BVyKL zD1*H6_Hug)rdp1|Il)HHN-ilL_*F=VI-aB7b2d+*c0d&Ox2zy3?O$u;RuzqL>c$jz z_$h1?+MZy4mDSZ0qg+@mWTcwJQz#g!xbNwChPjJ!8@*MtZ&=AZ^?|+m;H}RSQ>X0} zZa7(7EpaeA6*pTflY1ocFa+6yQbaVyG==%<#mG}6e1=Tp+p*>4WfCj%Y->4;f{y@J zJ%|`Nn$_W=ZC`R9!0DBpf1HLxos3w>{c`A>w0FN;8w58FIez2L>Ozf&-@$b}&~4dW z2#9uTc)MpC^{2w~7u(zhr*1LV!d^Qd?%p2d3bV<%_u&!Z*C6Js#+;jDUrFqKim8f} zO3AqrjBAilFd{aRSZXr<61g#0nb+p_A3B~j~xgZPH-tbg^6|dAIqMQ3k(>9JJM|cYmaGExY ziEX1w|Dz`(VKY}^U{+c5=A$agq&}w@!p9b-KYMRSEu@R2E zOhFd+h1U}*GBSlzi?9vAkDE4aTR)bv4~QQMA{xX7epX`=29!WD-FT|!drDl=WdAzI zj0#h=$jxIJ?wL$7&PU)>Xm-{s5u*aV-w4l!poFqXoIYhgf~bOgOnrMlmQvLI+_-sX z0;Xi3Lq1>ne&$y+r>`fwbyFhKs#)Z;!>%xu=cI{^&Q5OIba%bn*Fk2ZVI)aatF-VH z$?Rfw^H~77Fo~SOx8mS(RztQ@moGz*i_hn_6UQJi5@-DcW`|)o4W3V{V(y;fTS7f}If<`k4 zbEROSWXY?eq+AOtq*7H~O~p~#B3|l(3!L->S49U}2`egn*nK<6%bY6Fmc#tyM5_A@ zCub9%?c*Y;BNmVBT(**}4tppa&sZBXV|8mw9&0rw#emIfVfq8dfJ{y5VjfC3VRBh* zfk>_ZDd}G)5n>~GCTTx%Efjin3$re44?;84T3|7g^!pgpMlk$LvDJB9ZwJzx-}|yo z3@Us>F1*4g(xN}=hg&82ZV8r(H3_o%qEbAs-x*zWqxCFRBfTE^Ij=&Oww8~DcvQ!Qs#M?;_Gp(e^iREl679BCq5=>nt zg5@MY@AgY`eZb*l=v3?X(c_QZ+P4T{RzW}zCSm~G2k;%@MLMJDitf#lTwlE><5wa+ zeQ~a-%1bU$TZjwraArHtDMjKvM7@MCYl6LiOORwS&;b2~ihGzy`yNL8Mnjj3ErY-m z>>OcSz8HHBVQQ^0wxbZ|qEfJC1nUvK3VPl!1Rr1{`QLy4Q$5El5%Uaf;~wX}t<;=I zkkf_4-M-HHqwULP>5s>@OFT6J2`IoV89kYYEod@+1{TJmmvmC+rNrVbj&WDj2o2_LlWch5*wXxL?>7+U z!HS23uZmbZrJzd_)sh91bhzp%k9{cX*5pm8SRHLeKcblszbMIE?KA`Udd{;QDvF@R z)G74)(?S;ijD4WL6YCqd1UmP8xNt3FZdoT4(2?9v#+C@kNkU1R7+6wpXotM{NZLO$(Hz!)){Y2f*hxi3@#=9LJt^MS<4|(R zLuq?T&&+(gK<5QZX#Ew;Z6rr>$8|iPh0gCK;!!NE;qP`vG~-9@cpjNnKBsq z6{-|2dl?-=Adj^_IKRvXR?l$$2`D1ebXR-Yc^3^hk{E+p0li}2RwbMFuFAtN8(K%+N1*wcR3P!!5G~)2{B`t zUb~Zy`0zv(p-$cT;X0Iqs-4Cv_yEr!M;t{8U&^mMaJN0Zcqvz#5Sd-BdXgtO#MHBq z?i4pXx?~5IhOZ0*oX(@a3AxP*l&zY2LGYHuA1j7$Iaxf`p!f2!eXgp1x>godU-#Sk z)A6d<^_Pe-&sr$RhhZXy16`BW5>6gfj-8^#yS(8ci0=Dux+$(vngY*Vj4(11AcR-her3)?>q z0%P=%yy>(48*!BlDRm?u ztf{IkpvTRPR@w9K^k@-ZDe`>U$8tLVP^j2cemOY`WW98h$E#9X)ov z1KtRm6dCL*NXJO`O<$GN;pJhSuO+zG<~R8NwHRt36W=6XR~OI{2E5RS!~;?6?|YsR ze-fR-|7^4Ti`#cexa6VCj~S`gYtBnn?-Rc)sv#2bt}V;*iNR`L(bl+MyXYtLQ%=Tr z#YR>(y*H@7&4W0wUz%CUwpZ5wMkev2QEJL|@!<(%e_m z)q(Y=1--ndf4#*x7S;*>^m!%%<>t$*OhysPWhDxl0tiDPZ!+p3rTCVqJ};4@Xn` z7M?mk@!khNPoWC8874e9>UOM`%(pkF@h$(B?YItHvG=wJJ+w~LekVMPiM$jXM2CB7 z%~>_$v7cJyUiliwR}!s}WP^$NXS~;W6LsxJ!0;$cYEnh1482Z}Ygs3@A#20!SnsIH zXXa8K^nXFYTqA4S@pImn!i)k7&xIS2-d(;QjrC1$+W@dbr}#kD+3ejjFX1HR^$vsv37zTUnTgr&c zI)+DBcZrFB%rr2%$8s*eN9SX7s*|dUFUh;?4)?8(e60L?OZWwe*gW?n3guHCoKSJi z@HSG*?jfBGJgilJG$K*81JK;>iDIFqK!0r~(_3-xQ_surz%LKp4Pa}S{P8<4nUay) zGhKAr=mh>88&4_6wksH{9v$CksS-6+UtJ1=!~vhWl^;kV9s4so&s=Z<=ZgkCPXOcX zQB0f8kt|%}>@snF;V9MilvM9y+d&+kp3Y-_f0QMpP-&Zfk~<@-i5F)Eg-dB=J>E9q z+uQu2rlkGD%k#S%U)uJ2RaGz_=Y^9e@bQt}Gl)V~$gW&QsT#uIfvTtA4d^aRuFf-t zhwkEIn zvC>|mQ7AG@_bPQZYNKvux^q>%>-Eob$IA!b<8Txu#4c|JDJ$#!SZSH3I7%^{DY=xG z`^53&?nAriJ9LpO%%5Jcy}1q*Imdy(m z@_yA-p?Tv9B~2#;#;d5Rg7-Xx;%?|gp*ATZ)$2EeG=0AF>Mw+wJ$;SP#Ru>^J)((X z*nwmKL}okAo)OSCf!OB6@#9U9;;1habDH-fVk7k1Ya?joXctGdr}fKSkK5)jVeGMbH3ffc5-2*#fGCk;2fW?Z{i<3k{R5lmp*ASH*I_}uim7AQ+L}_lYwaWzT1hm zE;)QXUy-H@yp?8<*+9j)ad$Ga_rQFG8%zTY=%S4`US6%BCPPSMGn^Fze=k8{HFzEkWCYPS4YWY0lPJDfkjPS&)RtNxRRS< zWGCe*=9*{kY+BcyKKcgRneGhj-q9B_59bfXW43G^g#gq>Ak&Yq4WR~yim$)R*p%;Kb3z8d~7e?TLjG~!IkOCRPD`F+}UjL2Pkzz&!0TwWn1VQCC3+jyz+tb*JpHO03tT=SO zdT{wa-UDNmg$m3yu=5|jl*VW~sCmPi@72_r~m6kh$~5nlw8+^_ReCg>s8x z0CPAZU4$9|kTGMTq3WbYz5^Q`Vp+v409`6VwzK$Rn+g$QbtnM`#XLbs(-zH` zHH>sF0a+^uayS8BAN_=5T|YquRBB)kV}fp)r$Wi3KW3ho1{auQ%8ARg%<~0+99o*{ zYMIE$wBBE2S9YVg_rm_iv-!dYlJMq(cNl&&DhwUt4>13tx$}sj^9NV1Orgrl>vT6C zz*8ogT2rhX(YR-E|9MbFdwZ&+OZ`LkKO5DfkJVDTHqPt~l$gu&{b;M&H7fN~FMX4D zIK4$C-R4o;c+2B#HkHB!$*@He58A={+d|8pNM02C8Eu9-j#^v zK%Zs|M1U0F?f}~QqI~!k9wE(THa0dqAJm?K5Zkt5Vru(U0|Vpefpm9d$?ga!Odh{$ z9Mql~JBV{zy>8vOu{>p-4ip3)ma>#EL)4KouBtgnJahBf?QT)BOQq7`{yXwV25@}7 z@78_>k2_y%Q*VCotN+!S1FaJ0`j0eAn*gVgp~TU_H;`P3V*n;JXsrne0O~%|APE`H zNrZU^ls+bPY7!CD&u%n8+YQ(>pkK}58N*vkil8~fDBL*Ou1BC3ryNc+ro*0SBX62A z;FzDzG^`zqPaA^}zIhe)Imp7DbFJcLDCv4=D)}iYI7-BUX)j*1E83p#Vi~6!57-%> zD-M`S3JH(v=A)AFUbZWEZ1O;DSNioLwshCB1@bp{p!><`LkDWVP@(~hA!H@KG;8bK z0qx3AdxF9*I{>nvDIKR{Lx=9tKhsGG=L<(o_sMecY7x_RA5-}rki`0Wj>S8HhJ(yC z(N_MGO5jCCiAuKmY})}BUDiO}y>}>}5&%)`xg?1J)86w($wwwunqRv%fs0)!**aLM(gR-?PMuINt1Kwb zWFlk3rJdX$FnfDD)scH(@ZFqARZ~`k6uB4&<04nLN>!3e>}tzx#zJ8_pS{pn+COwR z2<2+MGTKWIzxgcDa7h9TUGESJaM<=3oy05;v1E6QJiIG#MDY@{E+BG*=owcbTtG^T ziD{9e-$|h!(3YUH)a*dwz3lF{=l@>14*M7>l+Ti-tvL8$( z%*RaW2>W6bMxUy(FWnp-9y#w%@*wAf<-r_SG|{~YG@Iegdp-_DWNnRai?IT-4t3CT zM;LNc$yG?3St~hBu?NrGub;>{5!p%6Bmiv5tv2;w$62(ut{hO9ocNs)>O%BMqDG1@ z!lR8;we?i$Sel+qqVY;bbb01#G+|7%K0~qWXS(u|vm#jg)eQEFmJafY1+Fedp3U=D zmww-HJEF!CzRNR`?vJKglRTi6LqwogVB=LXEl8SU^+Z0#YH8aJfS~XLp@s}F0+43N zDN^?PImYLA9^$P-=gm^|AvEF;f(HB#g;5@=!Zm;ldT-csp|%v`HE8s60k-?(6;8jq z^uT==)z)PF;;vJ>lXNbLpM%kOVKCr?X|oWt-{{UTMZGhW(@dhOSu15>`1@Jh z;3aJ8elRiU8kQv;u_0G*a9TDJ9V%l67`!eb9CzSR+zVFi- z6T)u#E_~+SyN*GEoxNide7G^jjCaNW#V6TEs_efrz*L^Iu|4TG{9v!hfwTKBf0T(u zpA^>2%1rkC{n5#kv>ns*=sF|h455c3X!=^eiM^at7O8$LE!*w9^~A{UeB}N+@g>II z;47cB&2M!pgmq&4s#D2DgC5&Ps&PRoU0=Tw#%R=8i+j-~sN9yfA$1G%Bgrg(%-bT- z?5DC(B0fTK^&am zXC)LkIGcSc;dksM?kw?in#l3+-s2EtoNO&9!vAZym0sp|>aj%m<-_fsC|w#tYF#djDWrSsMl zYI+uNk&Qe6odHjit^&#MVUmgWH@jK8d zlm*TO!tMb~sMDtYq47yTlKz>duJPDQUFYZLL5u-W?4hgUF1v&Zv48~-#Sj8~51Qva zZ8N}5=LW7mNIN5CU1&-3u%~8CQ^?G@f1y20c#UoRv#kXC4jDm{UV6E?lLiJbiF!GG z-@;we+|-r>{g~HAw+a(J*lo)M1_Y-m2{TKtuQ6pUOsR!^OpuLCRCw3UXg3wm^yQDW zy)ZOYXnOuHx6$OkPD?h=xbsUKb%O}Z}A3X+4+9`Ngoy@h-vB1!@|BLPa(wEcq1H1}vxa;11KIoEyz3#93T z)+mT}wd(aj#*79Ix%ohEJq6l0NL7oJ*FOE5j);&D2u}B3dY43Gw;c{TK`12!Gn{R3 z9u^90u_owgSlr@tbU?&5eS$DxgrHs(;@Tr!SLhD9RqgT;>HA7mk)AZR4UJlX0D8_? zg8hN*`2lZ|gV7w~eUy~~F03YH`ar&qOa8EBgi5uDxXF1k^{PvE@}abvMvDtrSEH7c z7ws&o--$|+Bk)lYP{WR=Jj}7ITpjHnZIpR?rwZZ^r~oH|I;L+ zj?%OGwzj#zX2jOO0lNXc*A7U=5sYaJBrTaakIQ9;Ly&C%yB1ZULE@Yj)j$xZ0{V~j z%ZTs*+Yu>UWi?KTBTf^@4$8x8Kr?M#cN06Yv_lD^Iu&7->;YrvgId3EnIx!HD^5y; z^OvNX!E_ES2VmTR6q14GI_h}Cc>(IBxxX?Kz}GoESw$Lmpcp_WiueNnJ12x{44}La zO9sggA@(-lqzpULweUl{Gn0DhaDkW;r=;N^)a+OH1o(;8zUgUYj2H_*>Ta-e z5SkV(>L!?Q|EZ&W0ZZ4%`sCnCE3hGD1j(%*P*|-K)82w1IsE3j>SuoQc z2Y_%IO#M*vg?B88n61!uj)Cs$1JL>S>3aO!)J)Z5)^R}OH6zIc!~=vjXEU_IAYStTIfl>lnp zQh=eg)%}2Ea{w}_38G8TWq*YPt&rp##DM5^!aRXQ$a4FDTm=1hO9w}surtzaÌ z$MyZfl0Fa%Cd3lw| z*u}@u&|&0K{!A^JZ&!Q#w8#i47`-NdnOH+TOdNw4bX(9Fv(1ARJn3!cXTPROU{#v* z`oJuYLx(5`6d|GC8iKdE!vvz>nE%wFl!8H@iSp^#MI?e0vjzPO*=EhG!)cr_lnRPC zMD2^dn1-Em4|ug~x;g;0WC+wROzNNKQX%=AvqMRK8<J*-d~|#l!hWq~w+@q;U*D zf|vbHE9!{epu7W_1}aklbP&mw!$hCrIb|q!pvhO@c|ZnA9w}SY_8gk?!lu?-qYsFC zDm$v;+hs-u7{sL-<;%zA11d-2_bOwyESpZ#r9ONl_}JxnuH@$ndozXZIAD?ZcQ&oi zpdolSG2LO4P>TzoK?InId4zC*x7GnNWPp)B7WbR^(f`bXM8g{vDjnvc0rZU&=fL!~ zz&dI@c?K{P&|^>xII-l4X(;1B>ab-+mjdL8eK!tAFe0=yR1B~a6k#Qcx?yCX5{BJ= z?h%ORwfwdR=r3N!^=;50d)QC;T>l~8D`^Z%I3qt4Eix3B$$HPu_PN~4osiY!FC8(b zs+3EK2@w!&PMaUN%C-R-OuMn}9Fyb6S4UEpXtbbKfJot@%~0;8FmnN*bh&e?EllP@ z^$Xy@BT`r>sN_M19|ix2-U?(l_1%&iZ+$ycIUvT2fzgS!E}(@5y?=V}?c@?N&3D6` zve(^yy2H+=?<#Qt9V+bE0$Rs$I1t&!TU|g_hn!X%g9P@oGyqtN72gCs4sgnDPQb2z zKB^pg$3FFs3vQ*bTG#8j!6&CCJDX{r_z6qvx^VcU#DoTKcZ)Q=pQQ^wH|^VD;hky4 zmOVbGG|{zbm{tM8sPHpr_U_6tyochBXI0Mu*9gE3JpCdpZo*5 zKYlbm*RH94j$XVbiU!2NdyW9~FtqPM8(0dt2Y~#66w*YsMW4sv2v08nV$svnKOOc{ zLQ@Wkx(P#7$$%W5r2%rGH1wA*3!=eXfFxCKhE-U7t#)O7D)3I{mBd`Hqpub`r+D&d zx9x8*^Ip!?7tDglca3~9y4W31T$9UAkpKtsZ5kW$p2IExj@aL(MMfKbt}J;|t4u4u z5B};C8H<*S9BVmCCr-=**UbxayA}`dEj3kCvxD!+bD z2NbWxZ-+}hcyKMFOgvzlTOV-CtVCbX?GX{__j4z69QjBdX5n92y+kA7y&3fwSgH+@@Fo|D-34* zKyBR&IA1LA9)xM&ScAS`c%La7+s?YJn85St)9ap;jQPhOHla-{fAYhP&zOa*X1rR< zmb24FJ&DfIDo~ntzKw}ClbkGKz4v-`mDFadKIiXb=HbPKs8HXY%^K%8veQnx0%vfT z&10#X0Ci>35X%rGk0vD{coGB*32g}a8?WJon@?TGW1D0FS#^IVQk26cYPd3wT{x1H@mPkmb{F|>e!Qpa!_v7uO z+akZY`kFHQ)P83xjXFy*VauF(T@R}tHUIi^s_^lx7}bdS62gl$c>N$#vCkDp;u}62 z@qZz#%M-K+YBI1M=a2?y>e&q*h$v_hfN6wMGSyM&G0FjdK%9IQN#y}#A8}C;k{2+; zdR3<>xM@Uz}lz8$T>BjIbQ(WW82oU^VX6Q zg01pB{VNRtEb}|4beG6Cf}%ildLJ~`H~^u~43g_eI2vdpaOzb*eP+x*{p!^#Pl_b! zi~83uOueh_U)03F%0T!d4uNwEnCu+j55Yn^Nv%wp0zEk0a##!!7)j)vknEM2CIY$+ zv@|!Hv@%1|+f)Bs@vX+)t~mdAL*A6QW+*#esh(fSi^MLVDhzmEo5 zRJG9>G%Ry~k${bGfJs7ex)&cMLVyQGQ$MeQ8-Ut4gYB}03gCy*ptFInB+k40sn%!oHh$z^#DBvurwxIoUG226?w ztScjvT`nI^;|VCZ?Xb};xe~TTcG~8A6C@;26@5Fptr})6QcgeoXQaBZqiXmG_oZBs zJx??J@DueWhjXEs@uL6B%{9);ST}Q3E$mrdfZX(o8COKw6%%O@hh2=%iGC!mqvBEoVoLrb%t1icU_R#K$am+0e3xduj60H=-AXPn?0m0{WH0M~akyq#MMn)nrzh$w{>^neZMChnzyCUK*+K!b@< zikw!q{q)xjkeEP&#KAct$=qMv<*6p8zgQ(R-!N)CdAGWf-}RW!WJdpV=(}2#3_h|T zT3p)dQRTSu!@ml=hPoY0KB%^peIrA&EZ`^KuSB#wGmP9f8YxMPWPlih6ynKKI{Y~U zeWb=C3MW7r21!%{L1Dx?xgzwOD2$(_Jzie+0{E|tw{GQIjEuK|Jkt`IxPt(|yySr! z@2Ok;a^67LaDobSEyOUu^?K~z>FMFo3^cN_<_r*m{S5OC1kenDeo&l6g$mdT@^G{e zq?^My_1TwgnEF*~(s)NH=hz?@Wb)NxlWCj1Wt{1y&l;BVd_0U4;|dRMF+%2GsnzAb zdJZCU$Xr-_+Rk{ZwaLdY91y#`(=i~UW0ZYz`~$0km@iFA--;#_lF`qgc~Ug?)oYaO zgnoSlJpA{20^FdKUYV*QcVw#-`a`-a%?3bG0?pcZkoVwjZ2)8`aUR_x8JkH}sS{U$ z^49N626+W$`$U-Cy?d^GrrwJa6oDzAiH&8?3|aiyJPkuR;n+7ptqzrO z5Q7QWXdoq%mCgbf$oM=C%-f|pqeX@84#GZb{7 z#W3aN#V)6DIxhWJ{^bdkpClAs)9;`4U-~mR;9dO~HHVN1gOP(tFkMP<^2u&5#*M@k za}=gX12Ef9ISuq`@Rxc(-wR>=0prb_s{tD9?BIcst|$`I0-`sJt&f6wEfW6#2?JYW zs4bi$(5Pa%V^CAyK8&aT=e%sm_uQL89U1ot%((1}C1`NXCz`%ox-({#9Az0??{`wE z|0k4n(q^Xr`rRzw%Prp|>7T`r|1Cy!EwD635R}&cp~lf8C_i3DeV9BT-QeX}m-Wiz z09%2R`5W|B!cUa)G=**7KIkX4Z=0KEA(9ui#Z<-cT$09-g-czYItSSxtbWZFICOQ-jRu6B+-;3~VMLe(aXItL zT)3a>5WyxJ%<*K>u<}u>hR*Fe1OCU$EovY2;rdo5H z=~=P!j($yT+0^(??6JV%Wr@!$!xrdKL*ST7=$%j6J4|0Xg>9H>aw)if$xEc3-L0XYbd);Zhh;=vj?wtl^Xyz_!3$J_U~N`B$K7OHi|?D(iGnt`u2Eo9 zn4&)kUwie`e2^2B0|WW)b#a~}#^;w;fMZ)Y?aj_;WpFei=djkZE>QwbGX*)nL~NfX zO85TGBo`I`qUXDizKQ*N#=+Tnjjq?(R_isQ7p625gJ+wQa*i^RRDmJTZ_9br*V1zF zA#pg{v)u};?wZV^imUO8(^SIyy{qFWp$-+Lo()y`A068qjteINd}`Xi41SeL(o)nU z)k5wxBeXdvXg@~G&F8vuiPbXk+}PbxI06m`56jC04(YI#hZ6#7n9An{kfi+2ZcXvr z6OC-BN|isnheeSA+UT>RU-^3g{=WTX=`d;4}MF^N{xCW zhQE*Jtl>j_G%e`=LLW_bD*S-**A9^s(soux9#G2>X$Z68=2SYKIF&Y%F>-^UQHh_f zJCF$4kTQW{CVR~)0S823;jN>iRwjD+t-gj_MOz-D9mV|7OVy5uhIw~Nr!_~PFay24g0A&T3>CPh7J1Fz>?Tb>li4k>adB#1qC&=NaD(tNqz z18`73Udil#5N*Wshg~&uPDfNnC(Cb~duAGGRQu@qwH{Km%wy;e|`^7^Rl>Zk8;0-|L^5KBn`!#(_;g))nOG zxa*RcR(7)N2IgfH)^@4}uzMQlYE%l7a ziNsF7cet+jvggF^KF1NIzF1PXxjiT{Xh}!#qpmk_?kPW|121d1RYM6?P5qw_4;n^& zg=EgX2cp3yJ+5_)e4%j`=~(U`egFrYAEJBbgWS$uxVjx=d=ujCzHX zWz>_13fB0mn*vxXgL=FwO?YYvgG(nW1fus28%`Db30Is)ALi~2rE)(|_fE7m=ZQ47 zH)kV#HcGI3;Ztnq;ed;ZnDg`-lsenuR;BVX;~%D@_glf_B-jtXvLg_a)&2T`Xa&C? zN|S+IUzMn>Z%vW%7Jj7fk6k_b%{VVkicP7EG(LN8)doM^dqIH1bT(`-M!?<`u4FbK zEAvasU0F{tjF(vh|ebRi}t!PqVAfpHu_1sMXBV1ZxQw0srk(H8SRTFmAUVyqy(q#qX!l) z3^=HzXhDp7?A_Sa*Db5wbvhrCl1jE#FIWNqRMXR`gS9cEzuXO~H%UFJs7 zs@JNnkQ&q+dAU&dnXfpXY8K zva>&0|9NQwXweD7I@<}2XGRIQ$!$+j_Ei)5)Klqu)T^xcGZ>2<`tZLlR$ekioqY9{ z$3ZQnYe&kb2ASa6l?GP9O*U4Htz5(R1d}5Rl^3%HPRpzAUB*HJyVfTbGhfXasCwDn zmz4Hgy&z~Z@6HTom1m-NlRib~Wp@l6iFSn$(ZqQ1H+H-A#Ph@R_6Pk)_r8PO5uJ;> zTE`7B^?Gjo)T@4_kAQ0bNT-6olPNkuDWRiH|?G6Q(OGzp6+(}eb}r2yU4qj)=u?Z zcJJVbaZ5~M;`=@MINFh(K<|k8i|gDhcT0S3&JZW3JYjsgk#NE)|KvPTa9PXp?_`14 zeSkShtru7X|6h1n-NvPId{Fc(zogM$R2NkAk$%c@-V8Y9~%K-;|6k zu*%ww@|eH~#Owzg``7R4CC$zpyxMqC!ra*7h*hx6>u;%Fqj&;;7kHMprnKD`OwQEv z8PO{t`}BQp;Xji^p@Xuh;3*?oGd3|yUOrVERjBy+u@`3^iTW+mD5;rwI=N@-69Ipg z^Z(r}>}u$JX_p$^U=V#(-K8=y;LyMO6x}~HZa$_bP}}c_NQWN#<-TVrb;*Osj*{rqn+mhV znf;8{unYRVm`RsVY-ER%x$j3uULU-ObNE0Dtd*aeNZw?cj!~rSP~@qM3bNez*xZ@; z)bm&Qj>gh+e${<5oXqT7lI{I8o6`7=Qk(yHUb5Z6e;1JqdlTf(jz-WZ5L5Qy-wm$U z>lRYhP>=Ep&ZBBuI#}^5-FR+HDt7f{0vUUEElW8`l7@c|MX%6{@bdURQR|`MxC*D{O9wAlv5d-eU8}$>8tUJc)oikfG_1!o=TWefP0Ll4%`-P z5$@g>4c-TH{ha#DuW9nTnt3FrURdE`_rhwfyRxabnz-Juy(t^F%hoZ56sD8%Pxls) zU;*vM_iz3xQZpIceQ}+S9nTfxd!t=ExG90Gm7%ogov~x#MGZNn2(HO~eEI#DW$$Aa zv>Ud^ehspiN@mne{k-~*^Y+imm0k<>tKp3H4$`nh@E3BDJ`K5@rEt#h^n zS1Ru%I!+P_n%MN?`}rzSE^!?C@0)MUs;6pYO%1+Snq-pYKkQ5q|99`ssiPqXg69qU zbft^JU0KeSKn(m4rBAlSmj4xXHfdJj;2tlQkX#u{@u=ddOCi_@WZ%qQ^g;Xhc+q8v zGjCy;Ysz8zHbrw=Etl)XW;Tmwc1^*SovTX9AzDm~F8-ild+L$<{^A8;k1|(& zy@9&WTca8~s--1tG=i6TH%b5s_ero%^sXKXvl4$Nnu? z?%sm^v53~}gAw(5a>5<(!W|U670vpc(Kn5C;x3UG)s^K|rqE_t`jU1CIdzBXk|!lK zPvw{&W)<4|B3v$Xn%=-^BBpeDA-&MA$IjN;<0aJxssnA4QZ1cz9BGLS>O(8Vdmo^d z$FBcxu^Z+qHiYOR1f7q=$S<}$|fD=SduQ}vNiy?gDO@{=`1=KjAz)N*@K z-1GN$(H?liSpBdr$>=wJc_~Hn?$PNA2B#Wi-6!w_1#^iX=}R9?9PKjUOe?~TOM7~Q z;Gc7IA4RiwzMnZ0lb$axm`tw^PCmaSVjDW0+C8?@G8$S>KBki07c9=k`hZ>=Gm3#> zF(Eg)3zeKxpOPeb&L6(Yv;XkS=;42mKf~9usm5u*P-U9g#*5_5ZQ?JkyTUGRetw4Hj|J&Ku3}wJ@?tJCmw!o~stkVpEqaHre`A&PYz?M&!+Ep=26tvJ zkvFfg3W?B=Ra%}PHX3ShI8eRs3GW_2M~Rg)xIWu6-Kay+VZtcbx=)JvqwiTV-8amt zfXC;ygRNljpRM4=bJ<3AC0JspMX~h4oBd#ung(k^|A=ws9$LybR>*Nw`1n3H58{>5^CpFhId zy7!D-4@L=Wmn`wMYz;yMv2GSwhAC6FpbLG9hqyH!U9fUA;xMi^K2W{yWnNOC=em0= z?}A@+O?k)_&6}6|V2d#RJ>hyt*!n$K2RyZ1>p&wK^|a4Wm2EGxyzjhk9O|&fVWIR@ z`nX$hDTOM1xuHo}V4z>epUI=D*!^K@{HCovKj(3K|0SaU;%{yI4)3*!voi(oIGxX? zlkdG_YGv$E|C1x2-1(O{>Wr4u(rjxtK~G`QJqNc6$x_?Tx?c0mde#&DoKfgymmqVy z0PmWzYw8SxgMWp1Pt5aLqc7ajGPc2KdXn%_kD_9$Yj)0gZPd}md?CS-St|GaN-EAnF^nm>bYtc zs)k{YtYXgW32~o~;nGZuZ@pP`V}BJ0v8ymFHV?1yj?~ArV8U}Tf3qJ8nzW5jFr>sA zE#w?x#u^K<9i}~Ffgz;wi@g)d!s+G6YgYeo%jiEl17(vhTfg$)(k5j7yvei0T1eXC zRmexl+?!rO*MXl6d6l|rpd4qb3;%k4k;G6|M`^@Fw`JMLz7zZEftHj$jRzOcsf9GF zAOtqn-p=6J-5D!0Z>D^MBBTP}@1 zY9>a;764@=>xVg1vgd;K8BAtL0$3;aO#TosmgwpO*{WVL9iX1Ogy|OJ48Ay$L>v{W zc7>jnvxYz|Tg%gE^6ujK8?2R|db}ELf36?<^6oqXKWqKF>6-Hg4$Ez|`OMb!8B8W{i4mloVA`+8lbEEVl$E`L9bpdCMZmWTzgnVC6|GDkYLH{I|F*+YC~? zLl~<4^*UFJ=ZeJeX~r?)EH}RIz)&pc21<$nD-3b6;F&2vjf;VWr5*2D@gt!`90H4D zc_b;!11a4eeEbEuE1Xdd`~|k3xlj(6|GchG)Y2zu**J{7!e!?WCzfD}O>LrA@S%Qn?cST8m$JCFNkCQ2^?DWJ| z3gxyAHUyAV@+}ykjN{*x!;5P>Zqiq%fN3qv$?)zk%{tn6YshQ86MFWv)jw^s6Z9HF z5WGWO~;|4*3roD5RJWy!|~iZ5a6g^Fv+A68`ZrB-*$bli|UBlSAbXmUa)P$fc* zT4M73=jHM0ntFz0A3s6`gK3QE1V_F%jiGfK563k}j8%2VAG-HV3QFoRAU9zIp!8G3 zTlgdM_B8133nvnj{=&#FBY@gX%eX>B29UuR*p_vj#-~7AhZ$P6T}*`*D2lu z7{3)jSQ3*1E;~*LK-Ap7c)=WSKWOR{VyFW+VF`ek_c2JF5>3V#ojh>(X4;XvR5{@A zypVO4He=&{bnh5fNM&9wk=3HSM&ZtRap9%H7VOC!Ue4bN`{!^Tg2M@bCx4nPkUMM# zsvnx+%eUOAgNe>+;I6%X6NX=7T2n&{hQo+JrUf;skWkiSe!X4`Oq!!LG%o_t$bg^` ziJU0mZP!R4RjAGpYPH zImyO&K6%SUe~Tw1>AS&D2&(R6Hu{7g=NM0U? zv)MTA@qz71H31yYKK!za>Lok@{n1|lF^P3=j3RS>QU`PzWlT*| zXcD0Pc1lBMpcAmmr%c6y3n>K*Zb3F=^Q5#lrwweKrHOB0t{CTcNJdhdPYi@Dk>2k+ z7E9041m%HLMtSF%?JS}PQ_io-$t$@SIbGtWD^w0tpV_;+lwEeg9w7K=D9Du(VVA|J{V-1wdQXN1QS+l_6PE8z13 zmT+Dw*=v)@B|lYyy?^_&To8S(y@kPbLBC$!AYsoI5#4aJ+F(C$@ z0)2(2ueYdXdV33UjX(YJQ&uh&F*YJfC+>@fo7Zn8U|{BR{{lK z`_FQVfjl$$f-rYFmK^_eKiCuyvT2wsK4iBHayuOMC5NiuR6onzw!l}C=vDHS5EC={ z%W>cn#SDq1?O2TIv5F}oH<+0KFkC_EU`n zKdpI1&>#F#CYlR(2Ds%C7FepR9FJrwk%#~QhN413$dNncUTT#?Qr4au=ad>*Ji!b^5p3yA88ckJ&okGoxC#abDDJzRcjd9T# z_x`(mcXp&US0w)-OQ^$0V9>26kw}~ZMqEyyahf3Gh-OM7-O56EA|&h=YTYb~)CjYI zfS%OM$K+EL)nis9nO3~uF=bnXAoS=2ynfgMrg(ih97Hq?Ec#0`0C+*#U}!S&6er>$ z(-oEFxvPT7iC^rFKmdX1Zy>C#>z-U0K`c;EVRQ8m30IS2KlvifKnpuLNo~nleE0m&s&-PS;| zU0}1?62!aJ$cx8ggV~?_{=Uw34DP+=TPl$GL1XtDjVN+guR8i~a)MS$cXWB(H6D)X|fjOnWe9TT&icd2nj4PU~F1pT#Ag zB|K983J@v6Xsko&&bA}WT0~gZR*>dQ0xUQRZvb-?0*dr8Akm=?lo|h#3ELtxff^LM z`V+6=Q<{wqzE~zY3vwd~z`O~p1aS%-U0q#KOlsi-LwFk8CB7NkL_(5`&Ae^6dYNVw zyBiPG#Pn=64rK|7tQ>%%j0gL7%MK!j+j;rhHrDqd-EFs`45z^vto;*jrr(t&#yyRA zXg~!~Lf9wJtAIlN=){X-l+xg;avWF85ckV}cm!?_cqrp&{M_zw+`7!7jCMX6??^NL z5|8C4_@>mvL>lQ6EhN%CeNpx!}AMv7;e9ec|Ut2G)fGX0omUA&q`H6ma3u98h` z@q}kiPW*TMmGqb2v^oZ5LM7(w5-QbQFFS5mv+HXt)A%V(FDva=As>-|(=RCSzAaK~ zmWOCX6Chd&s@oA@8zx|uj)5gUAp3l4hZA>xRCjF%u0SwG`rBpwni}1*tWUenDJuxTmeksLoE!Hm7YFMX z4*p7r3rmD8E8MsP@Yo48dl1l&foNU-qg8xzvb=(V!d4Z(jg5`Fy`-eKR%q4Gjc3k- z56!7gg$8~K3rmIVst#LgeWX$kStCuOHuOIyFE=1|K1?nApLDkx>E>H{aXt1^6cyU5a^zI2eU9H zs_DX8A$+9wsk$6o-##|Eu8oaaInU2B;5n&LlQ-w2(6^IgOHZ8*HR}4Cy1sQ#dp5QR zUhgRB35{s&Wa%!@cXCQjxBlpFzO4w+)wy)+KxnXGUj~>4?%a9=TG$}F4I#n&MLZY8 z&S5EC)oyET9RvTeY={q(^4BJ(3TKA~QHj*Jwa|$Q|8 z-llr{s>iFnfKRnp%k#mkmRtNMDBJ4wUQ~f;O21XO+&ky~8WaBVy79f;EI5`~h;cLQ zpFG|JE(;cR0*s$5VDga_MpNtE=h1LkIEJN(Nl7;K935kjK|X#>{#*ycSYeN^rLGWY zRBOBTmy%>04x$N*sc@AwX9a7V$@SuQCw)57{~Ajy#-=^3)->EN^?gy%0Gt8XQ>1K! zP6TA25$82~dm5&ZHWProliwpz zzbI9Dt)Rla^D2~GM$>i#V*K1=@s8s_O zFx6Uh1Jq*?s?^r;XEYwpfl;VR4fY(=>QJl}^SD9WM4Z|v7uqx3vZ4GHb&%qvV(mxd zhgiz}=liT5_OzVJ2r4{*8poRZ{^Ys9@kEav!?@5`lq6&R!bH8q$jG~}nKDqE!9|Ym znRN)qxBydMwDGLu)la=F!~04@Do!zeZK|Pp-{Mv>JmBEt#hJ^ihi2e#qn?KY^gJTD z?NSCT_lp*Aje`+1>Naf(NMD@>`&#%n=L9`o;ZQFEuufiOY-2fGu<)rNTTtle;)ZgB z!iIg$tiQ>dz0}lQxslUFx^{KOy)!klUe$jiXTWN|{_J9PbU_b>D2TA{z>scf5GwA- zXl|CJdVCls;&OQpe)&uC@PW7pE8H|PzRuB=%)6D?gU0*dy4A$dCDPtl@U(gz@nJ5_ zbc-|B>w%JGi>?5vI-w5omp5@jppQ*aycs{X1wrCwk^}l%;mbeJ)I7<+>EJ~~_FQ#ZF>x=R z?=?ROh|Wduf9p0HpV}I|+1^%|W3Tv5iZ@Elsg8&^>0F_DS^{G@TXJ~D1nt&*wB14f z{j6tIRRGTwpuT4N7Y!BTWV-&H0q-Vbq0Q;=e!9ygp%DUCr%lx7LFY!@et+eJO z1cz@`b5szeB+JF>EsFfte0VoTO;_)ao^qDW_>gd*dQXW9=Na>7X5j-kl0#yK<{oQN z^#;OE)#RAi7#nNrezrOZ1pUTbI+f9D<5tti%=uk*s4)IPQP#A| z2elj=3J{DMM$=D1P}2^i90d&xGaFSL0zz=0CtQzk@s~BePo0*44%~$5g|xmITx2|Go1LQKe~2=h2ELVt=-0qoF9iA%kf9lp zS`OogI}F_UG^wlC5?v{jWK~NeIJMII%ui8qV^c%1`a(+=q1vveb5~x;ZW0q8(|tn} zVBYpyQgfAKzmzo>0EgNc2G$~SHyRDu*4`deX>4q~RB>L@05P$^PBZ<0dg!Ya3fJwf zu7D3_fFv4SHmG)Ib}k4YQvKh)3OYjG698G+!sfI8T<1Il1W5A(w9ge}WYFODG30(S zb!)R>)*Q@05l53v5Yrd_DjrTb5G4{Ut(?l#j^f;X!N#D`@WWoPjtXkB^ihF%2%Lk8 zu>IX$l)JqnQf)HZy0+L*yiI?yJ%hFC3Fg-YW~MS+yQTyz!yHgxMT*N4J$Y#Wfd$>} z&oDD!0!^w%QtKcrfI^}Hn6lkwc<8zAC0Q@GqC|DI;nXFk$wsfgsQCe9g;|}Ly;DS# zx_640E-|UnS(kmy4pVpCyY}4=aBXDeF@GW3Oc3%UKGk3hJS@Z${zi&(i#V{`3kwfu z8(kUiW)OPY^ufKee90lvs)$Va%+pBiORf5!G6+<6eN1e@WygM&;QH0v0*j$I2A0MZ z>YT2>G>}bd4`M zs?h6r1#m{m)tL%^)ZvbI*|0j5!ENWQml8OZw%>61a*gR4Kzi^+WhEu+wMJ)xv&>K7 zkb7Q2P1JJNE+3$yG%6f&Pb^(z2v3z8CclQU^Fv}1#W@rEkLY|L6{s3 zRzW-ir=>O}RUk0J-i!uSdNGtSAhtc!;h@lfh72jjox<@y?jz~fN1_E4?R9-6&Byki z)@>!&z5~*R(WQg3thbRNrMoK0Detjhay05snXuJUV4G(bMn?#lS}Q{rH-q+LhgB3i zK-PgAIwJ5fFft--LkQR%fNT36Rf~Z*^Z-Z;g;3T+fQiQgwY5mig}quBjwaN? zu8M*HWib>WQjedz60gp)csjBt$2bmtV*n&zZUB-qBco7@iZwY3iTznE*H*ZR8$jh| z?{F~h9gddXkBt{Ou@7Be;BLh(j3Xq^wt_bY`P$EEwGzAzVh&l^@ z66U;a2zANw3HAfjSMxC?A%e<-;a0e>5f+o9>Y$dq1hi7wwpDf$(<=6xHwsgZS{HvI z*lXH3*YDS?o?QgndJNUipRUZL0DDS~5TL8Y#JG4{a?~bgsG_8pF()$sAIXrExLmuH z3~(@jTF?V>0mEL{p&r{gm6Mhfo7+6v ziNHu*-ufJBW9H*;(m9d0p%Sz49J*;jq}m0OC;Y??q(TBCo@RZ%R!E1*b;iV^}UB_N>)NGsh4NOyNBf|3FP(kR^^9fE>%=b^g~ zDcx^vf&2dR-n^N2XYS0sIC9S3``h2Q)^GiSDF_}s5<3UU6;9!}s3!jcQa1u8;6jmd z9JB2HPAK$-_;g?ZiWD^njR`=aEz19PTP%R}NzOZsTtM0L-N_F&u&Nu+@Y_zUf@g2B zE^)Kb$O-G2u5oyqu4>w&--KerNsNa*;y5)2!>QW_Lxc5nO7}hJL@cAicDAOz#oMi6 zTs<%OZQAWy3GQ32v`+5_oQ&t0_+JoU1IkY!2t{Z9Ry9_$iW)HTo3{!dUetdtvoPLOruZG2-|iYAPnuk-6_)iI?Fn4@ zMLNk)k8Ifalzxjp5W9N3rTT_V((w~lj<4td;CNK40hFKp_%i;+s2eH;5V@#Nf-=#`z&GZ=-bbr5J$fSI$PuU8=1#YsJrG%v=eS7* zjN|ow$Mp82LP%Od>J@oI6HRA+#Gnx7(`#k*AMtq=ZVD09-}b|Qm+Jga5;in+0cl+@(4{{2V3Ur7-ICFN3f3vrg;LRQjx z3uA_ankO}}D8hMs_Jy_=NFzby@B}~jy{aMn=-%TA)sbx1))cWOXqaL0ruz5`-HF?ZWNB64o>BrJD%LQd4VkdmX;+b$ug7nwFu0m*F+G&-tGD z4cJ-MKMwjHh<9{*UHBqo?x9eR?Nk)KcZzs==;)+ZUeObT$adF2(-qji;6G@ziusY= z{==EJ>K0w%Nq7O?`boP!W6y^ynC+(0+ zH9~ozSR66O=3G>eUKpA}+iM+dcu`mtL7}#G)7iadXxkgi!;1c`YPcOfv0RcuY1{WJ z#ixcc()N0}8cuw^aqh)kaD#y>GD9ZPj`#}Dt3o6W>I~M>G~P~u9J*XmpN+J-;8SB- zI=KFRnNgp1pFSOR& z0A4{*Ehr%BWOh1K)@Rz@fyJEfKg$uUcPzb0hmp-AE4k{IV=2M;-F$Av=}UH_@igD3m&-#bI@uo`(9_Qi#wUD>k=5&D%+j6Cr&20t)D*8 z;E_?mC=RSAx?lokWJ!a1o&3Q~<~v)(rEH%0n?QA%_W;S=V;)pM*=2;ri4HkAS zZC-XuAUuyNO)_u~N){uc)%b`F6W-F|e5Fh=P62xwE3&R1$B%O4WrS{TxvUx@$( z93e2%oJ#5jmo!0$=le`wy8Mh>8xV|^AOkMktnUwGO(W0ltd|WnQ#3YS*f+<{lt?jM4KS@?G6l1Q^&UEBuM3(}vd3-HDN6(b<6oRi9zq6FAEK=ViNrp&% zn_xQ(y@MhnNoR8ZxtU~Z_)uZy@YL5qqmB{!YL>97$smP-^zFz*A| zGg{4hZpWWIl~1WycRt6~TCb^F*Qe|R${-Q?dC^S(yia71mKPD{|8@M#%d>-4B3a8bjQKZGoi}m#tKXZ2)U$92!goFlN`(%GkftT-@M@L4(5>bj*#DbXZ9ahgBKQNQV3!wm=A;3fcl2xsL^&gj;K+z+d z(-ZcRQk0+D+{KycWo+k zH&asQ)~2|%bkNP;`IXD9OFArvJ|&8*q?gFfaWiHH^p6|ukRNv_KOfS<<_us+KwP`T z;o(nW)H5G6EdRZ7uLm@@L>&8wtr?8^vV}Q3!#B}$DI0;O@ouD3^UD zYQS}JiPlqL+sRxRO^nxH+vXHqF$}lD4BpdD%~)LKKtWlV@nrO}rAXZaAII`}RfWRQ zj@#lVCmr(pk|h&`0>D^@uK5h=tzSG=#S!1F)MbD}F4D3IwN7kAACV!SE2>(W&qL-n zorT6WFScmtVIUyy>SJQj-N>t_lba7b0oq9vpC#}Vg5nus0@$I0_M{oh#k(3H7iKd5Kq>>&pDOZPC zs;q-1b;mkar6py%CjPrD4%qe7^z`OlWnR8`u>%`3}~O zX}6G0pE_2bV{&o8c`1y&AlPUiMQxDd%zcm7_~9e%qPTwTt5=;QG()pr@BCscr{h^LUQRBtM5%4rPT9m$u1RIGI4|2S9&a0AASpfssn%UJoMZ(9{~#;$KHp~RM~PospU-pk=Mt`t`ew!OjDBRH>8VCu8E@If*TMJb|5zhA#crr z)knvV&7o9ruI=aWbD8)vC<^2NI|m2Y1E_(H^hpNI6i+@u?GSH$S$bL}IUyT*s&|gOL1U7w7H|#_Kw}THn2?OxR&8zyaEe|!+Nn3Kolb?9Cngj z#W*CxOgqBV{lWzu!E-19n)@u$8_O?v?|SMzlGvX<5)*8IWOJ9mfT!^z2# zlX^@#dIVaUy4-^}aO^L`5Wm5_JOxUG@Mwd@~kCV8vPzcq20eNJ6q~a!1mRb1&0XHa+xGDJyn0ubvVnkwCpO9+Yg`8qG2-aWyxQD$XLZn@Mi8E8!PkN$>|nW4ln-T;UP&H*e+#`1A)=q7UgnV zV;!eL=3};AbinHmbi=u#1YOMkG*I9Jvajr5@7}(f&*VcFRwT&nl>6-!noWP?_8j_v zMAG44XLG$_sfCG6{P-RG$}j>%c^7ASW!O+Cl>2X}pgW853_9{%rtmx?)Nqt{M+EiS zSeltOI%;S?vsqTEWvUhQS}xQrrCO>Rcl3l@Tts^|4e#EOJ^6(Vx?MipkW72GsR*hn z$50#;6EWz7G;=s6P0`&oG#5|zk1@iKsO-yT2}N*do;+_!s8`4XPtT97O>hpFZyqGI zkI_{2Br-I~s_T7*OJSd2R#M9@Hk^OM$z}VYKIH;toXmD@!@FTEBxgER`WbHd#4+f6 zb4y<4X6gL*hA<{XYiFZv{+PhOr!PxC>-$YvGCu*9aJrjpAnj!Hc;ID@O-x%P?QOKx z|Ki7=5&SMGbXN^PK6cY>y~V)0e3@#s;!(;kWN>egX~+D>@;J4bJq{Hi`twW|_4wL! z&NMCitu|xl6E(^&FTz;pCV(WLn_=ds*I*6nma{L8hUHNOp5mv-djwsB*ygMTjID?`{AhtT)6?loa4>!0+J$M>_w*JbJEK%uEd zjPZF?O~i-KR_*K>X>d7eZLYl!r?UE)3xyR$4;>vH4P@ote_y@WnQp1aW2S?%6WyaS zo<9AE z)5qkSn}t}VJYe)(^Y_GIJ3eugs(}5FMe?ZK2icDs>tjI;&L24}$x*P-Hb9W(C-E2R1>yqwi+hb4%TLlgn7cHts zRranAK7o`Fo=&(qwjXpJ9q$WpUV+LvAzRbjY<emolB={=Bx!-E<;{L1$?!ohY6P9r988$w3wK4sP5G9hR)g<5sP%f4$UGwGnc7B`^QOk1?9A85DT$gnolUOOQh6Juv?1Pvz zVbDE0IXY89C{^w>Wg^9pfjieimZdSW?gMoQoJ9h>wLg-me{RcAJP*8ufKKIP-b z+2luwRS_0;qbZ`Oph+I`oWvfFy6t2`@xoHrtd%nhECVTYhQ|TLj6afSfe@ATATJ(x zhzFJ}gAV$%j+0W|)HCrIYdn4dw^Zw0 z8HIdZMMs0mjav)%sU`K4EO^Xyag<)E70)i+ULW|uo$^LDUTwZSQ;Q~>FBh%seoimE zQx@%QBc3T<@)M6CxhLbJWJ{MDqc-gT(d6n$CwS(X@CCP)C#&Ab!2(l$B>2`n+wpy+ z`vCHNz>3u|xB+|;^B`pwNH4QMkMD#S`E~e%)U2$mHgqA(kk^2;BNPB}sQ^#Iwm-chU%Wdj8Pxr7^b`7e>xx4p=g1XD02_Fp6qsZW zg*!%OkZ)%j?it2P8mM{7-MTh6fbi2*H*u(jF6!_@Y5@D`NVMa2Gw4%Hdu>%zG`s<1me~ zn+nyQs~IpV$SD0IgfPP(%LJIEm`)&$#V8Ep?;fO|+Ec~WVS&hrJCLI+=u~vz8Nz^$ z^6|QqJx;E8OW^s4)d$DOA_>;A4Q}4~{0{foQQ>N+R;Fe)w{7iqRT(XpLx!O}9b9eH z^%VDfGGSLxlU|UZ`mHLQi`6F*>fOkprKhS17}f(Pcb6$&?Y~slBkt=s+o>>jZR}rn zTbZxf^DLUVfIZerNPGl)=4pz46)>GFNbtKzUkJ^Q0J-Cuf6E^ztlq=OOG%aFi3E~Y z1UFHkz$pD*lugA0t%Ck_h+(Itp%DTXdb@ES*0)CRSQ{o6NGiVljQZz`zmbt;J)4^- zdUq?IeY5&$!q#Hsq#cHwQA%1KquJ$T6|Rh&zkPSt8GRgo%w#GAbM$H#9rv2pTu@#d zfWdY^9FT@qK6se{!7A^LZgT`1R|`hR<_Vc2OwR-25H=dDssuK;ne5}725Ek~9ESa$3HLz`Fk?tlbc6ndPVXK52X<=n8>0oZN$^)8<;Dg`y{qYl54F2#oQj?q9)n;Ko?}*5BP+Vt{PhM~|I! zLLnd+06D<`u*&KiDFvH6;Q$zqrq9jI#mM8v*sd18%X+Xpjfcagb4|_CFcO-j8_Q3^ zVpol>Cyl%3KCE(%j!w)RmN)BJuX{*pu{v{Dr^b5lR#4h7Stat3SbUHVx8E^D$}D$p~2u^cv;=LZ&f3!dzP?Kd$A@SF5HnNPtqHtt{*LjBH8SJf;TXIhWkNJW!3 zmz{oydr$5{K2H)`!1RhvXRn56HfMkLxxLVZwp-2k;W>KmGDx}^V{DnUuMPI98goCA zrQ{28-#Q$UOUk11E558_TD$&3q6MAR92!4!wC`2`ABTz!*$*? zEp)KUwEqx|;KK|PUW2*>2PoxSrrVXO?iO40ZauwV&34JuPhe-PE50v3c`FP3ICitV zlhV?Sn#*`xv3`K5?!N_`r2>rZgv3OBC{n|~q#O8%X3zDf8$%_XeO}`c5J(?xyH*vy z18QI=LWd`1!r)GuaF-4jR6d}lPHsb=c|FylO~sJ+uKv$PbiaaRK&IQ4TrcV6LkU;z zON;F?j50<^&Ob^@CHT6?;uEoI7#x_+!8=gA;cD_@yY^Di;BL7STB>U{XF6miN?Tm= znNTN1#?3*y`opye)*AiksCD*-XHO4#9MR)pi$Lh{F|#(l$PX&A zX+}7(!Qz!#&<mhit@j!*6TI9^OGTA*=rGLY`74WzXLVYL3(9Wm z$P{bCi#B8tyw|UDp3CdTD!9Z_<>f`QR$bX4E$=k5dPYO3uGn%OZT6n1zoVEl&(5(A zi{^gQitb5?p9b*9D4gnai5k_Mrmfm*lqH-x`vlh*t77GFTV}R^JDoP_@Ok#JT$Juh@gRtz@LrhcBsMr7d zf~TO)u0R5|Awc}|9mLbE80130tI0S!eT{%X7Y+o3CUFP(b18UXPy5fQ0Rc4y41ojU zAnAeSu`xK42gE^sVuR7}p{&u-_w7~sl1hcWN1x?J6+%7bFQufCtQxF2>p9KcR_AQ5 z+}nSl(aZFAbD5UQ4i4*Kf2K;V^r5}usG8!7O+)AWW}>w()Mc7zGtn?dp8+z@qN5^( zhbkvdVQoIAg$4v|u)iN3Y?dPt36ZANM0ODHhWMY(glb2%fo3cTWFyKT7wiJs^F1K6 zdwXBv+?~&ni5rN4VXbPw8p2V6>q%Gvw|#v$>I33H-}hjq`?SuG2(;tUglF^fv^Vlk zDepCUctjQ$>AN;N83c4sUpsplp#vrK!{j3`Dp1`c01?hKx}R83xs3@=M@@)$>(a2P(Ip>!e?+NK(tm2 zBEr9iSCgV^R?1BiI)2Q6k@D&vXK5*gwxsR%@sq_n@hv*6bEik?6OH!z>x2xh7~Tlq zYgVT_Txfl-4oz>Wn8V+pM|GLbYOyie{| zeB~e`PQ@9N#3^!;m5(Beej-On(0_TwMEmn1_bO|i=;HKh1TifhNIxQ0{%;)!MbDYy zDRmI3@|0LF$*7GEb(?`rx6q3h*tL#|d3x~+{RQM!Lr^BC0UzCtbQfI6k!S?kdo?!s zAj0|q!i49D(88lMzYF6BUL`TQr`APl0}`8GR&A`WXD*ETvmazruD4`yE3fR`+tHNo zzLLoiuUgDix+?)!W1vvLq3dU08t?=d(^OOL^Y7(=3cT)59`OVwXm<*QR zf#3Qu4C91U^xxmNm>XRx^FN1_OMmymzuKXb`f2J5J)XiOyd>$yns!v+#*YS}#0`>7 zn?LH-fJ~@U$b4F6zH_u zfjH!hS;FL4^M_e9`6G6#rbEpiTuy3H)I%wbrHc-8xPL*#og(=_CfyC@1RHelz1hW% zgxcB1Yj(~f#J2xi+Jt;x_Nd#)5AY05DVv*~ouGeeG%Wu?R1-)m;Ocsr?8)Oh&`?Av z)d54Y{_TJZKK7=R6lqs+dwr~;hdZgW&^3aIr8b7>45r_!=3eWI<2jbAw__(vSR9yE zver`b;&qA=(j2<+T86k3qnmdF>B+es-Yk(gfrXSSxzC?L;GT(%1R!!7@hp-O6PZe3 z)RhDZa6GJb2=#n4U zdMJ?-_{S)Uho;po+wcx7BzJ^&Ftj%ZTrLwiF2r2O|NcJuTiHHu7@dl6J8>%79E+0_ zF`sWMbfjB)@~_X1C3@c!e7HhCxVB8iR|S|Jmz_}tOIjaPHR)p|n4LiG;Dwk&Sxsp^ zJHz_N(%tj_`}A}00{xf^L@>OiVz4zSIAj*)C?G4YMVAcDvO6zJCTw*N(yli*(tt^*jT+DfLycD*Afl-AhXCSdShy5~2r@J(Y z_j-)7Tl>(h(MIQ?G{40%sl+v5St*+@y3OF3LxrR88tCcl3!Y^OTg`O#YcM#KQj|oG zD8L3w>t3D#vVDL>_op_{)U1~;wccZB4y+m1H8K3s z5wBS7ki3=DoscbsIvC2{1SdgqpjQl!5^%DL8}Gmry2arHeQ34;VbXYhP7Z z!CWkUjW?~dm|rqPTmExfb`!M!0ov|N4#r`(8A{ zF2l27^}i?Ml8_B_NRttW5dos^eVA8;F@UMT=^pjYa8q8pP<2%Xc(v|+x}*Jt7rnoJ zCl4IhZ|u&F6V$`F=M#n_0jEv+UYVii5=}z~rlVO_l(ZcCn5A>(BHS$20h4!kX8i@o z{K4YjF)`|RH{dC_&e=o06Hi+)``~xN{f305r1!vxwdkn4O>mW5?Z)m}{C2%U+5RUC zW3f_(Ksri&)DV%K4H~EisV-+Pb?_W4syM+O075 zS`?9VFzPM|1J9KIp2o-w7%g~z{MZKaz&nCSPB-YS31AKj9=iy#vh@$@@gu}?fWDXB z+9%JD{6`*9Wa;ZXtk4mjNV|l&dV7!XF#^plWF|>NXS8+A_IPlUCp4pbWB19bN_yX) z9C7pGYcrXEhaVgil$@3Zcr^nJn;5Ihoca{(43vavG zdiWcM9TRqtV58=gJhU?{A}!m+Qpob?i5Se2(~_2-^A3!gV$x?rbnI8yRTEDJf7qZf z8UaOIE;|i=iS&=tx{F@6YTG7&MMEMTmnxoCF3_P5T!NR&HFo<*CL_*@@ggK>ni z4F->3fvT&Ia@##SKF2(0z$_UGQ=WGi`+TC*i;PiS+N(>CeivNJ?CNKvM*}^l!N{5^ z>lg=jDVG~k$4B8^KNk%~Ye87ybhW={Iu8bQ7;pS05i7~qZUFze*Fy`e>#NcUgYVA~ zcVWcYzxJ;uY%SRiV^aBWcm6vle6C4hJuB!>2f_LrxZZamY2OG+G4~l2(`!NsrfM7c z*YouxLkDfIIGrm_EGlVP}0xowV2J4?ToKFpO76b`y_t5 zmwb!q1O@lF#)MTN&b=>%ocJIGwOGhQBg|(cp8NHe4uFVY$>{(DNvL@q^_3Qpb$+1Y zy?^}zY*12jC;ACl#QEdL1O#AYf%huiw@CD{uTy)aB+@y-!qDr&h#2o+i2SQ4B5OX1|A<% zAq)NYcq7tXpg8)w+ZZYb-4$b#Eoms&xvhEh@hBB|(IU7Uix0p|pdL20jFLqOz+K*P zoDm24qF_7J-q&mK|B2~+T*xy6y34Lb_kP~9LdaTOpH=?={6C#Jvp(;$;at4^vk}B+ zL&r+2f-z0bjW4A&4d3ba-@|QXzI``ztcsw24LDWO%Kav;#}s0hCHQOyz#=IHAvFvQ zElB2HpuuXU^=ZHwz>4Q?F{5;>03jvxJ6c2r1D{k9dfvdt#Rt|{qCaX|!4O89V0PcR z#lNI^aHJ0mXxfSP54Aj}@$a{Km>UJujIZcs9A9c9yyVSD1Qen_Cb2Sto{~}!tnHWe z#ZawEMG}4C>c7lzC=B^|Jpj$NWxXtsNs$hN{g#G~iNpJB6FWI)pQJf0v@lm8Qm*p7 z17p?r<5ni-AJ`nhG5(Gqv{%f(4bPFA+^HKi`J#?;spcrGO2U7FRUrp{k@#Ton7YLM z%8>nDShbcz9)G!0LB_8bbo5-yWav!6 zQoNyrx!~1?AYc~Hzf&yw3mCT)iu7BpF5w*Ao4?WyFqIJufH5R3?*J&NpevUMumGrY zmy-ddrHRIV-w_|&1v3=%5!`ohVB$$88C_8-KJ72B3 zn4FT-6^`4|y7>0bS+tcnak8*`nU3cYSXwG}FuaO{5ytj00$IZjCnW+pB7y_J_Wr`% zU`3wtTl%7nXHna(mjfv!p zOQhxxZyQ1DzaUmNX?QJJ3O-rJ2O91)4a7rOc4jS#%;2$3+H2@l+InY11$A_rU6!g;xB5`;J(8s417Rn(G{2+In%iE_y#hAjoV2+)7 z;ay_WjPB{;4lSnVkv9CSK2Hif02-*I|FPK#5QjNy;l|jw>Px%SK+zRl+Guoy<`5YFe_bM3G`lL{z5v!v9CJi1- zrpjcS%$e zk1H#ZoVVYARSn8ipbV7cf*t{2WnBmxRmfI3R9 z-?Tr-1hXm_pIs%PMwm8Bq#591laQ@i-q7BdiH6OM<&iXS1{JO}gX3AVax6MAD)U?W z$d2>zC%dJ@Py_D~GE5;$TM90-_aQclYYJqPUo@WHz08s4%=(^uK+8g&5Z?Lcg$>EX z2TtE%8xOB8bF$A4ZZRxuMYAMlW)Q8EQI{o}dE6D_lr03@g18uL^Soe>H1_e8`30G3NPL@cC>;iAH@ z1ilE5uFxTlj!IxZCQWE0qdO!dO*kp9;zq<87lN$I)_`t`r)VylfO(wp-U?=EOS@Gl zhx%Uijr1|@>_?xqDy_v=EnJb@FWfVYo<4b8!oYpLiNwnl|EZ^C1|4*VoQg^mA%jX9 z>yQnv^HwGK8dQI&Ai*Mha~{>?@t>5Dnyxcnekmni-ESaI`a8#Me6o;lpAWR_i3p4Z zeRuo3WzgMSH5++EK9_JgehZxmT-@)lSXm(2nW)-!--;RxvKt>ID#pxmQ$1oI$VKLB zbAbo3k(Ba%O1Yg6HrBtD)6k7R&hz7}i%veSXqtj2Ij}e9dt+fF*M05W?m8wB23oR+ z$v_k&sUp+1Ido?9U;CJD;Ot5(^ey|IM*SQul4zX9TWpTX9MpG~HnmC+VxfDry>XdF zZYNTc<26-q+;tyFmgX|mnRmF|dR}QtO zg6bLM7gmpXRxTQ@@SUP<#?GfJm;8)vVqO>3;ocV0I z@kwPzS2vh~#rBt8gGv!9R}MS_BXpkl>a1-13n;v2~h9D zOtl?$m~;EeM3y9Qsen((=CtzHegUyp0Reka$3mCZ2s!X(hZ-yJ4w7wgneZItER6lS z5z4UKX_QWBSo^7xxAr zoVb4o<F*1b$u*$*wRnf%J@U# z_}Mo7tI6?nZ)6!x?5U~6!74Mb2qCpvECa zjL)Kq73xMmGd>b~r8|4U>)|!aQxx>|@%}Y!TDe8o2oeBn7@z@(Xbg)f==CYv#30Jq zS3ee+s#x*&Qd*{l$2*Q&T$d>*mDTG1Pp%BzOhNwb zV8i`#uZPr(zq*cWDq!#my{s7rr3J^+$QI|?fQ@gv-VHOFo}wy0d(FSnAhaMdviPJg zKnwC{IY{tHwPq>Y0RB_wZ9{KrF7tKTu)HbD=&9p^f=L?+3Rutg1Oa+#I}r?y%zU0o z58!-Kf87FIH+3mwBAD~O(r7ld(5$mN+I;j>-QSWhLsC;TosZ~ivO>O32N0nsz&_pZ z1vuLo@0%xF$X{3~ZU(BY`>SXg=6o;R`M@G;D)4(%{d3$5g%Km2nxNCUKiyin zx!)B!WLveEK3gtyNQ4YBcOEf;{2E{OM$u#fZge~)sJby`B%ZDkZRL7l!{Cp=G2x6S z2f-B)T$iq3;2U^z=po|Sk$1Scj=LK>D}XFw5ErzVr|+163yh@E6i);Q-*3E*3Oi?S z6UU~7D=8-uaBEC5hiya?WlT0w_ad!q$t4{GR6Y*bS_JFq z&y{BAN7_UHyN|c`&EBEBf_w}*AtJUuOrrdRxXE-49*Ty#IYcCn+N0jmAasjI&iJCCG`&`&W!B(xiwou*Kpyvlzrgu*8&r zA5=kFv<_q6{RlgG^1p6*s)xHcj(^5rL`9Z~*#YT*^US()!d9?y!}?+xcm0Ct?vvxz zC8j8=lt&KmWH9THECsq)UfvY>-eUOD!f8~i&7y+&MFAjY0l_8IN=u%O3tomFKGV(c zKGDaXW;83pb z^3*%pbrjQ|C+_=~|D7ewTt0MWte%HY9(IyTozW*vXw>y7-cW?HHu=G-2Us6$5N&eYE z`ez^(=crcd5#~K?t->EJEaoBSWR*24*i;|heiz+oYAD87<*dA!pq1g&HEnxbeSiG` z@F&S_-#42hT)zsvKC#5soCOrn%c$XH!9lu+hOpN3j0bM8t+I9jhsA!aU)K@jyKSut zasaND*5+>+5oC)-kT|{ zOw6}i|5{+E0||(Xl%`Lp{^vQwjv!4F14L$4;}YA;Sty>haLe1zY<6OtomkzTCWT33 z@~qdRx7>XE9+(6V0U9uTzp=YG7}U3H?~1lW&|Y`{=z1+ir&LK9={VU$Z{TQu{BJY; zYuWqw?0j#oTt-~JCnn%)AwAIr@fU%NG*3s*>zWmgiL8=l6z&wkpZ4YGW@E{-9yPh> zP-LTW{nUAjpY^~ouHqJndtpg)>|t5%^L{fcdu3~-eo4!N5n%L#>TMPn{h#~CH~!l^ zTg??B!7}D>tOQ5e0sVD7(>gOe6%OzBRdG7g6%2XiZD{xHHbeX5&Gf)iky^2lEHUAK zRgen|%k%n-H?CAGDw=nKm^7n!^9;hv`D3$DlS-qvUVWR@y4j7KUM-lp+#cyxgYIt(ZVNHq~oiL|mVmq$AkgPx)cYN5nv~WW16Le7f05cLPq{mY}j{YLe0*{utdaE zc;MYNnkeD~S?6qmjgib(E}JRVB21$pZ)F7i;G-$wDuFx}pC6kT%X(_LiI9_;?zJYI zMK5e{=Bdx3mWj0Ewn-}5n`rOX#;rg(%h~EDypd1Dw+fY(#gOQ`F8!>co&twOEyvYG ziA>B`Y`X>KOD6u53DFp{Wf05**bu{k{~p!L3(2QOeMvd)2tryqq4p`%*M@OG98GeB zR%KGYQ;9>k-LlNhMtP&BRz4wkq{&x)fX{$-yH;+m`Q!PW@%?)#n!9oxJ!=fI81xd_ zRvG=!@Y#H>~`=8W2aXTr3xH1?85| z_eb3@dNty@i5t;frhOjR>XXBfH7ps>fbg^4YEt|V09eStYr_GZDl;>{9WpBH2dH9s0On^f+#4@^E) zz0c}s6#5xJ{)?Nl)fLLL0D`l^g|Tk zw;Lan=Q|XzIIMg!E3@Y>K{w@5VR&PksM^`9F>~C}RN;eKA7rIF=X?Yb^sS#3>H2_` zo8>*{B$@M5q)E}*L#fh=P8*8H+y|ZqCS*>Y#`{p$xT6te33|&F2;XC4W1X=AxXt8e zTwb0zEm8G@hB*RLSMpx4DgSwl(rQsS6 ze2NMyzSWNx4*2&$E4PigyD=$ioKJJP?k9LOt#}OdX|3UhU%qlgKj8yW(kYuaHW5a| z9P;8K%`a`&41fAPtNCE#7N$_&+1dXqz&B3)deE|Df!d_|17RavaYIwjp4 zUE`$mT}GNY&lCINRzJyovv&DG>8x}q#&Q*n55~P&!humc(yV*Kb>GQ5ee?O0WP!$8 zZnu&sIr)F%*<3eI@NCyNwq9-E7y(2e#mKWLCVOOKnTyE@vKi3fyAxNp=>78fv#@{t zh|@ctYNul2#lLj-%!^q*UR8lMO6zI0PvG^d@H(**sZ$b2hvn(T@)7-_nO0)u`QZT~ zby>Qe6i|vytcx^}rPs`S8;@qJLhx=yM*6^hNH{ny7oJ5<0wz6UGF>DR-B3rhNIy&lIy1?xG=fmGb1ew8= z^palQt3mCDm-1>KtI~NW-_`CMV)4vg*d>~>pVO1dxAZ9x#tP~mrgc7R4O2~;P4@^4pR|! zFg{1lGF|%!B|*c>s*)P)Ys^gyP0D1oYrYo@`AuVvU=OPQV2@cgupVYgH)XG|?3iKr ze0>lY>c1k0^Gs_T$|sL!BWJ??-pAz+y3^@Ag^h1Td?UbXD)3sX)|wm;LIK7w37EGU zXCQz{%WS%HV=J5w3Cy4t1=eyV!PEz)-};yoCeu0FKI#0bOt#~v<2shh?h4a-ax<_Y zeFnMe-L#pd-vFbXWpLj1660L&L}{Kw)dbJv|9FFN-j|Gw3`Ew4LGvk{#C{^^Nb&ur{%xb~*&B7IEt);omkDqJ6u}wT#}o0>FxqoOpuYC&Sw%abKduLUWXY=AqO4!Ei-2CGe}ps z7Mk1cpTX?n;nDmb7OH2$dC*GF5~_3nXO03a1K4Z`6Q=-d({=LI`!^Y@gv@2-KGpdX zL)i~d)2oQ>nu9rKb$dyNg#ZV~gT7Ol!xoXuBWmmV&KcBEMF+h5rDM9zamU^58If8r znbPe{dKC~C?R?NXCW4CD_M9C_=ed4k8ZU|~t0V$vT(RFzxI4KxzxkUt-X-v$pfA_{ z!oKn1p-ZPK03CZXkN@Td?l{XN3ZA z($FbW?+d+n%ifH4oTMV|3sq!iP9TL0aIoZQS>--odCOgn>a?RVa#Ql_0cF5GP~> zW1!po{S%vUr+#?bdwjE-RliFBCcjNUaMkF>?sD&NuCAynb=1H8f~-CVN>@Lv@$1{mud(3$M$q?ZYJ*N8Z94^;{jk0p&pFe-3US@3;>_mzsk~u8@;)y$ps!8kFaFi}J z@}(?B!N4S(7_RO7Hj%8TO1b{+#5z^_c4FJblZ|wKS zdB|wv_L^;N!KZCfPUv*imjBe`&v7p{=ST!xdYkn$mL&!VJxSoe6d-~xZ}Wq`oBCJJE&2hS7nRE04B z401@!xKJ^(>V50XRT~rZ1vdj@&3Aq-hCP_&=qlQb4&SOA%S$OM78?~C639a9hHSo4 zJ0PQrD67M3QOQ=0E{jNRSR$hdH*_vh@Zc1IC!PsRtLN|ItC`97t-KVftz}``J#*9i zXi>%J0_-LB<-xBxd=%;&cp%9)LN@K?= z4{rEX=%R#|NK%wJ8uAV{N!_!tBw4BArM<;|r1?>hjSIEGkQS3{?2(+3pT?y!ozaGo zY(bv}RuuK!Eyb0i?rfw-tAPC<6px9guCW2wtftHKb~6)dnom?;o<)$XtopYic#VkC zXcK)!$ndKEjos18Nd<64qaM1O><2I7X-Nn1Nt_=n>q^q-;N6qb>!&Q7Sg2A@!zJL5 z6R2pNO+*f)N+ZVKPIBL6y7DYxu<)9Il=y2n*^PM60(bP(Ff=kQ(0`6QAZ_j{bUg%! z?izg?2)%(I* zDDF2YYdyUyaF2~e=-$Tsn-2^0{ry__kLC0*a>|F!w6ED&LsU3AA5Cml^KL8K17WCz+-nu@bS{pCy-#TyE0aVu;!WU z)^sAlLbxxxa!(NwB!<0=`h{6aR+9QSDWpKaWv;{MdAqep_M)=W<^#!62l2ci8yj~a zFWSqd2=7pQ$FGkPl8!@5w+?feruCiUj{$n7k4+k;xM?ubK&&nlO016D$!~C2rS;_- z(h5SrK?kIu=Nj}5kqZGhE*Tmc>iO|w_Wf!1&hBnKP*EZ>Ll6)u07DPhY`qIUnUzrE z@e#8ceXbmR8A7iN_E%oF>%S&r{M?%I3oOYEJ6&QWrKIi%3JD2)e;nveXTRtWG?~uy z`_VxtwYF3#9}#J21?S|P<7Q5Z`O+bK5Z*$2{d5g&<80Og#;v#IQD0TLc9n;JXw$n} z&jqFQA29Q)jaDCW@WmI|IsAFOkPP8_{cmJGc1Dv@=ETY6Kw?$|dK+60V(@+Y7xpH4lP6 ze~v>^#GQBhhav48J;IDw5JH%Q!0L2MreE)Z6{KPdAqolD3v6pi!ivdIr5|;N(NZ_> z50*YVKPa9{6?;1;vuUO=hu^ z%_#*#l&yLZ&^j=Deb6P(f%3yeU|?{`NKgU&<)jbUkZ9c4o0pBy=1QguV!-Yn#OXK!p> z_AB z{f+!geUXceh&SQ+v%8)_bc*=~CSu@pdC44i4W2!49GcmtlXyLF5>_a9-F3E>M+4$u zzTFAY6C*DTtV-Z0d-xkBHpwYeE#JI-o7sqdh&=4`{`L^qvoqD@k!e}j9u$iit22=` zTGU{r&6WR%Q5=gq^45{tlBoZ&Q_|B0;1t2|oJuh4f!bTxdWa)rfyE8X?H2(?FfDMI z=)V3p!9lnFGj)9n>(U)WDF?DK20?b9o?q5L%x=P%^&!|vz|UxzOrzS{|4vgR2RchX z5$xY%9;$QL=rZPq_D7=tOAvhiR4R;k2WVJwrNJz*I-h~LY|-`OUai(11Hx!VwE7FZ z(qE6Ai3046DcYt?CGH&EJf2oCBedC`48a5|8nuDHC^PV89f2bcIR^IoTOfd2|8s9IE|xCZZ~4i|h2gx8&UIAl^@j^hCHa3bmB85P^ro4N z?-!a>e$)jx;Mwiq7$=&dahR&S@2(F*jUNc-{zD%kj95e?OLGuZV0JTKJuZzSp#%4# z9lmkzjp8QcVPRo`>wXH~tg8n6DrDL$-c9biFp~c7WeA$k>wJRZhu{X8#8#@j>;=*6 zLCm%H13BCYvgeu6ud0S5Ifp;@dh|>ZYlmpg-t6L}pu$+j%zWh>b1t^e_Hc_=pSa~Ob zq?g_5-=k9KW-T%b{ma#Rf7cd<@tBt94r^@!%xV>9-g|g}wZ6AK+|D&bUV*skAU5G3 zjO;?pHBt`;6dzhdhlTBPyA4|#s-|&AR7T1c8hfZoNJzwHp8t6tKkDn2?`f&Rd5;}* zRIwhP)Z$B#5-vh5%Qpc5aaZjunC5TYr_XG!-}@KNlB$ZvRw-Af|3%kVM@6}RUk?T# z(x7w=3Q9$l$T{3Yn} zdFB%*_St8j0K5H~F3d z0RA337Yy3@+O=qYIJvi+;-9;c90Y?mX*Cyb-M($(F#4!slTf$2%-I6id+I8RhAcmj+HO>se-gTH&DNu-2 zPPyW>>vcN|Qk@GbTuQFC*CTWOw6)VJcAmpMi-%WeRF7?DVZrc|nX(YLHHW`|!U&B) zpPpabNKAY8uGi15WXKg}THSgsd9?3bHu8f(_~px&^Pss9c_H3sb1)$P_ZvC+%d?4Q zS1_sPJEvH+26Msz4y73eHQ2Z^OfU{};xpGryPaR_(hfKOaAdQoUAm_h!c^`3O9;aU zRZ|{uSaA4fHIg!bhg0ga=QiTh$_WvJGUWEqy!K{&ef^s6DU^PdX9lpLp1<`wSTMj$ zcDb*puKqH?@9@D5j7Is3xHuci8kie!@dm_+JV)pzk4!y@!%=9nGy+uAu=PtF7U7Y@d>2o)}f)s2T z?97fg*a>x;J>S%UgyBwWzSq{0$JFHwj-!$p(F;w=5KA=zzHjN=u=%d@^M1 z`)Wa5OShe!&Qi?9tjMRQyEsQK|EZsQnEZk+R(7(|>6bc~0F7N-T-R^6JMp(ih82d@ z3e@d~;0Iym;ei<*1yWgR$`c1|Nzmn41?z~>%&ZF2P*a!H`Yg5fSF*hU0&!3;i*p$bo&tl395M_eC z_Du*ydQVg6CJhV>fP}{KaM>;pO9~yQ7HT~W~ws?okYRunb z*v{en&T5I5+nB><{I4KiO^mfmcVls^MqJZy4A-+*)#gBc?7MbAi@<&HZL zfPl+@L609RxxTd^ZQr}B1A_(#!sG**=kEG#EBkzl(HT+idolYY@h}M{pl^NnqlqcZ z3o&{iJ{9uD?!ngBKmRkJ#ji1OsaE)n?s$>(z(t%QxdY!d>fW9{I4d$;5#(pGvgY~c zJ`HB(ytj2wEakgb60(!XTdr$Fo%LJbs=cd@AW`tL#XpwjBgBM>hwH32zNS2rv~pxY zJgg5|yT`~u&~?JdO1DsPwyTzyeb;Ys#9dRhzYlxW9nWXBDvO-vl4ibpKC9L3H#{3G zLQAQcb7gaCcZ_cvKaYJSYhq0Oav+b!OJ0eC@#_mVI_!YUV+_7;rXD`DFmK1NteIIW zE8iS+KCeOHT=%-lJ&M$;!#wB(iB?wXYyW#06N7ONw5ByTYEp!JBw~87iSYFTjPw%u zDZA%;wtZ)A?>qmXF}gtp97W7Wxq`(Lnd3b?&&SZQkvJHVfyzc+q@6k z2&S(@*5stAXW4r$h4zz?MC@FW@uY6Zbb5ZhMON6Jf)#)9ea)BI=}hWHiO9=}GNv2l zTW9P1YHH}nA}A$`Z54MY zrB)7v3bhmBZyZ0Jlhyu^bV{qpDoVs&URIXHVG2(Mk=}MQVCk_;J9C(stq8`xLSxmq2JkcXKaKKi!nBreK}CmB>{9eN;Y+xI&>?G%HY!9oAGZ zRb1QEH)6=KF=@_mlNL2!ey_HS^q&JvqCC`u%|t>{^cvaY0HVB$IX20Xsi4&J^>MpsI?iY!S1Kg0vDRi&D)9 zXx_a1dDgKJ+&^_gLA!p_uGtMv-}tLrm)hb>arD~*EX?#d?$kcsKq>1!kXFb%J`p_u zL8FX?`T6&Ixyp12iHL~ssrhm3t2WO6{IzrD0TSu#Z1+}LGsdh6?53p?QA;<(RSlB1!k})w3n7 zk{;03&OQ%$(?E$m!>ISKJ1z@-1dvseh;bs{xHZ^TF4kO>wAM5deEB{ zt*oq6($;9>DeAbneoWkj>L=FOYIp+_~d9^3G2p%W93omxufN{Me)_PQqEaw?k2(iV5kld^#=yHpyNzvzo)CT@(97mtJY zDP!zF%7&7CH`1Bs+V1TiLfhb&83a8dA)*S&C}Q(2@DbfiJ8)_J&!uAWM6E)-3JcND z+$`j^1fxq42mmmw(4ll<=MmvCG57bsqtWR%p2|`Ct7);ywlsLKY%V{haif|ROfkJj z8g$BCqL(3|8n3EU5r@Mdk0R>bXZ5Ot+icBMsG9oObP;S5(fIRHOTLI9MaTZr$bUX> zHWk;gjg*y15La0u1-zDet)B)5(-@DAjv^3IH_aX|!XqyiUy;mn{na0=Hk0sJU4`qO zT-{DN8#g1bDwn$MCtJBsg|u-taN$u*kqc5U)n4`*@6`EpTDfL>N(9M1p7WWRDjr=C z5ItSuo72E&rVEWUG-BlRB>A#&Je;u{QQ+`Q*(}z(h~EHXH)Hl z%~i8EAKg$7F$NYeJsGK4v&;VmRu@EqM zir46+tZWbEb%nbq5Rpo})?zDngQGsxMNajI5w#tV80mOuI+0yv-ru~|nRhPy=67rp zv3nF#X@1Lra@JK@zQ<3T?L&_;dPCEfp@l^t4BBL3W5WXhNot5~!{En>#leDsY9BEe zo+coB{W=pTCjs0H5BT@gKXTzxBDi6`MSk@uFcYmJJ$cfa_ooT!Qj4iZI$+rwE|8uv zm!!cFvX_TF&#M2h9kDBpo5*k{$TD9A!AA5WU-$%kR zheOT8#Dvw}-hS!wzp^q#=2e+oW3Lo6bME;o3{rN0J0&o-einD0iOkAAE^f(TtWrAs=0 zO9S=E=#I^jUGciCHz?N0x$Q4`C5}9lx7ziC8)KWVWn`AV>u33ME|Ck&o+x4j5Q87W zU=8tu^=@2TTy9DteEbGT2T-7fVFX%eo+dDN5br#PCPGL^h|lbntgHr5)F);sk||Wq zn_ta-Pkxzs>Qzr#R;^&lLKz&J8)uHt*6j939RH z^LC7Sl5sWgop)wkepI27SF5SMM#7;fb?RMjS;_wHu~jmq<4BeEkT5VXkd>Ebh9gv6 zU2U9pO;%PY8F$WzIP1etXIw19n`3s^JmA%TAfs+_d$nIJEnP^YN$;Iq?UwgaQc-hn z&0Sic4thYgRo7osP^;;?L)B&=gnN4W{(qme!|_2{K~}b}UZlr?z2UQ<1EG-$M23Z6 z?b_-GztC^B)0nQV*QjWK`{C;hS9Q z23jHIUvDAJOvP1p`9FM_olqs=P2S`Euyp0Tk9FPgCMVY@n>H|w8n-7);2TE`VtmiG-NMkFTxne0LGu>`GQ1@@RVTUU!|Vn&(+&kon%&HpKBO^B zHU*sZ^vRHYT7lz@#VST*BfOR~`if9=|HBGTPfwJwZb$T~k-%e<)Lpc zEv+16`@7%>RXl4@V!NC+F5Vkm(Gd>Y*JoWsYqyS<9=WD@2TTi0M8-l406C^5hM1%) zl$>cd3%6*aDE-qdvdc76W#wOHH=eR(5~!aejX~YM(&+OGyI$ESiy}9-dpzygR#J|- z?TfvTFi-j|!b5t}pox^pA1|J*{TlVVmeor+t^fCo`l|ObzPOqDHa0Pl%ZUZP4umT8 zwsKjRw5MS^b$!W-SjTI-5I2SxYgtn9)y6Oa<>BilLRu|iT)2n-)EoDRehugQJsnhj z)l7B0yO_lreHDwsz2kk0*wIu&IY9dv0%2ic5y8iVd_!~jcV6}3Ta$=?q^G;UfVs;^ zL^o!&$a+KediKR-X#lsd$4GZ@m6bfK%ejgY={K;@p^9O_G2UooL`K(YkspIA1 zCCa)_ygXLJ0C?C11k8x6zxxHUw^DNH5v+{cTY%d`^=m1Q?hy90JEd{CsWi%W(pl|!7!)iz3jii!e^=`H4xh>1fA?E29z zIj)>cwW(&Jtt5T3O1=|fyX(zyK}R2ZYsU;ZQXHft2f1md1%^)~{5ftd+TfSPJ=bUN z_<@H@E-j^mQE=q`piSg_nu#Rh39oRVQra~&x2M9ND5)?{`aZo9QjsukPh8`YOm1B* zW!2+#x?J99xj|ypY1x$9EuWw2tRTL$qgkr*%HPy8l?iV7&aas<7hSffy1_vzfan^b z3!E|Xlbxpo2}zJV$=8#oaeN`Pn!Zra`E^%nU}9!D#CM`r_|?m9V)o4Rrdnqa%7Kfj z4Q=UF!E~fgF&W{|t5k}y0!f02E?+bi?bn93zF%&0{$SS)!Gh_sNn{AAt44(FEZ|JU zY;)o|te6*i9p`QlYmoWGffClhWnz+uh<`Hh4~ppUp!>JyeqU&WHDMZffHxHf6Z$RU zdj7X!%skhk5*e;4m5NpF$A!9aX~_Gv#pIneut`9ly^gcq^x67~(L?}~ht(pB%j&ZA ze;(!-RNZ(9GDW(uYNPOPBJ+)G1(C_!bF!aXN?D-6n|KNW*E2Rt-%F1gYx9cwj}|p- z??}A#DWsV9oQjvnMJT$_dzcTOcOjrO;a1nvczjnVtYNT{POgge`1n&jYZSn>wzA5D zbmzTZ#d#R1UVC`3r>vsF?4}JS>jY^A{?l>GDor0V5>>MY;1qNZm#@q+ki4g;DF1RP zV!#`v$1`=NUd?OpesunF+J%&{j#0TstWEo&1BBfKP4hi!nw1a4pMU71Jg7cCu@~YR z`EgZLR2-qz`(tnKxsy1A4YnmC0YIdpgFpLcWwFD2cfz-+sZZlsNJs?gMM?vu-}6@3 z!YIU)qQIR)*_X*HEsC>3<&o0fGxL1H54ihhw+bQ#LDQF=hkKF<#Z#0@IA)g^cJh8} zh_Au%O(bnXQv1V9JRuOqKtZkLtX0uBL)~okDM;O95+!|Bp%fn+j!Dg5APXa$aSI~B z#PCy(n6J-LL0E9!WY@%Jg6z97@zHd_!6nX2!QBJXdm_tp`*d>hUw8nn;@@`MIv8-X zis)K_Q0kvg2_`>KS3i$I!7oMK+}_?E2jHy^G?y@$BFx9cWMm9}`#j#>-bt!#j?j_F z46~RQ3xm+1?%;st&M*Qtdq<|RV!ai81929DS3o9AAB z89N)=wEe~A|CccgwLN%y{62l~J`(`{DygWTpy8hR;>FV-9C89~G8V*urDbKwfZt=V zH7JL5LP4gdr^m4N@#)`IsZA3=Lkj&K#ah`!g2y3 zd9bbm%ftU5{@F7k$(`?HRU)#orZK;>mwRtmr0%@639)xFsK;i)*b_1ehJblLssnD|tE3rjm`=8KYVVn%v%AX}WtPjtDItHQi$Ua{}J{fZN;lsqDkWA)r~j zuG*Xz(3pXHZ01A=k&~KJtTM9Txx$6g1*wCH>Dsd-hD9 zfF`D@cA)lG>)Mx1F{Rubfx&~nsmc%dCZA0;+M6?S?GGcrYvh)Rf_LWV4rRy1N`mMf zH|CY1$xuj+gtxZ;ZS-sN6-(5f0quzx-J^)y%=hP|xCYKaijP#)7RdL~C z2>u1WGU~do^A~Ez;|>xFa0(BfEIG)EmKNtML+JQQnSJ$=-zI4MHRb+saCFFUQTI^M zyZAnNrd9Br*C+oP2tZ#j{Tqmn03tl=PK(ul;5grNxh;lI-W~=TYq7+{lhGa^*Ns0v zc1gF+0@Jj8LR_~~NL+zST}l1PwK?#c939~|BG6%HdC~T}0;lILpJu6YM{97k#gx)4 zE7oG0Xf?{`86FV7P@j3tk8^xkzp_F;5130}S{f@*9Xqo$xWu9D6&MNvQ0WhVDw8qH znD$uAZ|e+Spi&)A4NV5pGQ~wn&R)7-B2k78l3xwbxcm~e^s zw6bEj@HDN-PR3R?;V=C3uYDz_^K8YvyA9HW?JPNs*zJv&((`Z{7mtAMto}*NejUy+kKl&%+ zQiaSV)j#!lhUD2{F8D6&2^xURy==b)+7nzIx@*IUJL5!}9hxwkRp)+v(e(?gWZ$!l%;y-qPDgw>`%%Uba zJUsLOJw-5|*!kV?;>;hMF)qRPS2Eky1sCcf1*#aiq~CbKj#(7uJ0w7kw^)WsawzBz zw(ik}lSkjv*AvSb4Rwd9(b}We<)+SyGuJIBu~GIPo&)2#I1!9Xc+ z_Nbs9D+|Je+@Q+g;o)%=8Nn~I0$fslgXfSIw8D*7e*U3sqC-216s(K`-Yn8sy-Dbu zy#}pFViv{T-s=5^>jCH&B+9L&p2Fj!A}7~2{7R;FuX{E=P}{a$lOEoxONy0@r0Py; z2^_av2!MU_*UtJ%GvvJNm#!Q&k!&P&?Mi{bEy?4HVH+jO`qTGL$?i#3JHCt6S$^*O z6^!Kq5Dy~|0E$QO@njD#jUm0N4*MP%@r^#RACAYIO$x4PX1C=TPaY{Diwm6I*!MCX zyk=jzvR99~iOu))Vw;Sdm|B}{sWG&eGVP1| zhU_Db;C|19G}62#TbCb(_BSGkc*o=3aee)qKk#fx#Q>z_WrB3c8JE34v9Ju=C@GEI zGw}4}>`pL2e%4Tk(>W$Bl;hU=)!k1rx(b|MgIT@3P9^J!lXTXO*p#^t(1uzx+tyEh znNm(+>!rC>^$LAG$YkyND_4gPdSxe=P;H_ zb=DcP^CXaY2nB$R@K0|`?ZE~6d7iXl%e7t6%KbVHDTiUZ#gvhSeYJdZnB)PE#iZ1> zcemyX_FwBwym(s^kp%T?F5XC8x`Vct$9Rac4%0a?EF{uu2#|k=_3p8Ryi`Vl*WcJ8 zczNSL`IJ*P97;M-Yjc(uvn-4+FZpYA5OSj_eyES<<`Tt9B>t^%#N#(zdC6i@8^Cla zr;XoB|M)xgg%QZW;)yguRd3!U{j1egd)g9dkUZho3`k^$XuNMVs zJ2{(5xRA4camS;=rLHo_i+G$KfQvs##J&5OgTId5+6QwR3?lMLBygP@tybJ+E0>1x zO?!LXu@t6yEF%o4aNN77?ZjJzDU4~ui#w&FyC&`adb6{17ToIk&S%aF;Y*uG>}Uud zXCzweT^)>-^T3!>P*C^?0=G)a%2?UzUfl~I;%S@E03{&q>&ygo#h!!{-zD|dnm%)S z=GQ!nYZ`KY_C%G9j$#PXvvcgxSysKJ)>X!r(oL$=fy`tU|fIVT5M-3Cpx?sVK+@H-ey#sOB zA&kO-stxdd?Ck6)MO`jEMxmIZgA=GpNT`Bj*-&sw&&?1CG^n0?lFr^57$2L6j)`e{ z;XATU?f)^#R&J2-f;AbR@F^6loMFc|&w8nEP+fhYovbs4^N$U6NI~4u>XDxP)5|qiYHVouA*BE_Gks0vM{fPbF~{ zU?}SS(mh|*teZTUdwk{{p~qP{wL+sJRKrgrR$<_Ho``~MPk@M!6QPkAjYQz;ykJUy zD|>cB#IG`>nXJnK}Gq`tDj1^=wcp|AZL$)u=sI>bhr1*P0ZeWN^*1 zs!yDY$AK<2rqnV|WD6SmEEq_}^yWV`M!|24ryl+;WWwdcgt-2{&bZuPZ>x=4C*40V zdA>uo>j&r(nXx{`hxwlh$a01v!;{}d21Rxmx8p^7c?;YlNF8G#fU%FXfYnMho%w+S zFjkeOTKkRJ!pZlNyhWdU@M-+5iPB`Urma{$C_-IP_K3#_dri-}opJhV7%r6_U*!TB zc;b%moA~Y>V-e+)t!`4dq3WD6=6|1b1Ck*S^g$rboH_H)lTvYe4X&rF^(OsJIBv5K zv=UT_o}O*QBsdlKpKU%eKxawsN8cq%dYgCCKO|?4Lh$V0yx`Hg*|zzfOHY>d^_Gz1 zJ^6|8$dFKpx-Ft%L=qJ!g^`KTVNCONz_N;`F(yD@STYt2q(DJI^wc2*PJNSNDusr6 z7DOvmo#~AupZWezO5mMucEW0b{Xb9npx~lGgvE~%8uKpb@-R1N@!g$cRe+KX6(2U> zLU|=Z`|Iwv^n=^vk!4PO&!~>hkQ811kE=@jxqec(=(}&#tGh>3xHZ4@z>;RiP4ccM zlxT9DA76fKayB7A)yhMkx=7LMJ<}~mLuXNSU*AqfNq{Y1?O%P5OEuSeMKS=cWb%jN z7DJAnETRPJ3KBdoE)w1Xj$=2K7l8IRp(8m{PmR1~vYZgCMdPngBWz-*a?=|Op@1t*0j{*_ z8cKZm+Ob<$d*ynXsfbfoTH(TNuY(u&cI|WDtac`P<}W|vLRowGVK!qCHUk1smM(qk zmgnEiUyK)s1S$0|BuY@8pGL8c(VWh>y*`zroXpVx9y{E5r?mD}Fp4XIJ{HZ`k$^O1 zk^EoZd_VCUG4N{M{4RxjR)oUSFWfb!Yq^TML!A0-*&o45gS{{nmyQ>A*sHH1n4GzL z9_cuJg9Ju$PL88ynpdv+k13(@@yxK!+tCKBKdwTW9vkBe^pLHv-c4xu6v=snbxe1`N#ePQvmMjab z5eg3d-HxlPy;Ab3X(xU4q0nnqgfvU-P+xx7ESD`ardWXAG%s+|yT}zU%yggSUu*FL zQc@_we4_}uO?X}Dh5l_d{fFg0Pr%+(5pxMOxb=*guN&Vu9i}%fn9?IC%my`GOnNt$ zn5Ctx9h^w9T=2@g^tKfWCy=u_@5)`uko$!j7=qKH^zGb_TdQ{HizHQA*S0)dYx7FNz&f+Kgn7YILVn(nxJlc zd+va6dA204YfVjFU#4z?BDq_u>9hRt={AXs-3x&1)K-la;UY>074m$@9}ph_sJ`<& zR_W$DQmMmjW}qSm7Z&mZ;FJW;l9JmH0V8o9#U{jSYG?#ry!Db1YOT-@6aL8*_WmO% zZ-&C)PuzQRU%bj>X2?ilBAqeCW-}VeKyHa#%4fduor|tBSI(LI+>*&Uh zA)$(fOy~U!w&7tnjgq=A~r8&7= zu_9+~ZXVemC*?G?H*r3!v6Z+pMe1hUeOdMR z;{1yiEoQ|?Q*-VAWt;~!Ff_-70}&pc{I2Z{Q(h`vU0q^2x=?7nVZLyo;jKtd$wlW$DH7jCF+S&#C86=I2fV7zPrlOrK`u>8Il1exs>rp$> z+}wLerM)Cm1XownkX-;|$xRZ8*$pqr?q??mrCOX)7xyYT!)qn> z&5J-wG+RB;esmxVZu7<@5EeVU#m_TEyetW|lbgWu&VO*iO7$JKNxV1qq?ActeOLRJ z{h%0JYqSF8ANxL`DKA@>7tr$cRmBW)chsyknydNebZevH)Ju%Eg|N>qKkVd7E(70g zTXC8p#RXvGoI#!yYW6B#>2z{|lsGjVtLfJW8={!1|>KO2K#FS7Z zyxi*XgsokQq=kIJ|8#LAoAh#R4EfO@UaOHrWU8BNdNa5_;uzWU6vEw+?G z|7jSuN!dkKZodOVat-s>Qas02i5Z|x%P;c_3y!m$Oo)#lL8z>x^bpj{kU$WSl$4zP zR>Y2JtPZfYbJ<+f>gn%4i7+-ceqGton-3g@H=svt`6~$BaG^xwW37-bmwIF(_8mP} zGQRhb1*j3uFDe6L-(aQ4sHOsW+^SCl*5$P{}27L`yKP5Fva8Kv(?}P}z&cT3B3<}Db>2x$|uh0JELuNHoYEHlL++M|M%g{+9Yx|gI@3uCM=-3HI1J~c^ zd9KZuw*rodJ3Qofx~?)1{UZJ^O!9C1`DQbW9rtvnB|kw;ZgzGfQ28H2ix^$Jn0o+# zoG@So1@r=pCqg{Lw2po1FW2HzWUJcjf7@$yhZj~Ce-mp2-?%9HTVJuywQ_&Pw*D$L z=Q{5Zo>FIY5HFH6FgQ@c&aV)vxlXXOniT~{*8`B3`1`*X@BlF=whzNs5?4@qEILGJ?AcpZw{DXP6hO6qCgu6(P=QK zp`z0L-6u0EH@5|kg(-^xx zNE;eHVr@+{@tEy zFMOhjd$~>>$Oi_!(E62tHh6NJ`!@D|6Ww>3#rl)`l3f=24M$6`{J-uTMO*eb|#Ra^tAoTuJ1`hH2y#F&`DRWB*2nt-vfMVpTc%{ht8E|Me>0~dHj zphCW3ocYr`{qtF841RG>KaQN%{r^q|)HKIS3mI&x%fknp&gn37TE%%k*`yEH4C+#8Q zf4gCAVBj!(xYK0Y+7IvNE%wG6=$6}_`1noXpW+E7_sF^(T{Kd6$gJK?=p}wc)%ymh z8kYt)g!Odf+iamaEjirT`ppKB=fbfF7pKcWBw5c=SllyUQdF9D=LWF0&XXyDcdohlCa&A8v6GzL_Cmep%YT+~s8hKsEq@pr;Rz zqP4eg=kPgkF|By6(X;)4M)5G>PMf6;VO(p|3nJgI!DXV~rnc)WlJZJyjnC%?d<%GU zD@t><4ZG*^hQ`q8CuLKEr`N)VT4X=JpZ@yus5V-MC8c;YFJo}f?-YfI+{C~*1}p#q?K;pAdJL-dV})g`zt0wVa%lXu0pDV(B?gj^fiNu3 z{GYG2-w#J#P?V81|4ug^QgPTl#`vd2fr2TeFL;&aDWtUz3#v+L2Y@kw0XtQD&;Q4O zOtn=%c7WoWr(dCgzl$ zibtv>$AiU}Z#e$3Y|DGc|F!6YH2y*fwf)+7Pfh^Woyge4B=iNz2%8{okpINB7J>P1 zY_y))L=GccWX%t+pkoT;8s4|v%Ixz!2qsiXv9hoTQr9HhW6>(>AbFvgleQa;p=khmPI>Tf#RZw*0&xmHhz_x3+7eZ8nJ|i_Y!j+OBr4> zowHe*2F482_V=<-rQZA0lb@~5z4WQXx$DP&p0{i|H+4hF->|)51c)oNPFEJ}Bed^D zJ$XVCFg}+bg!_|!So@fX-=YhLppy|RN2Wk~A42_r#Cu<(tACo2)Ax-T7wnNq)eF4O z{j`#;mX*Zu*Ox{Hbf%?`4;!X)V3i0%P#>pf zsmNW)JK#E+fcM>~h(e=fQ*oDp!24NIaPZ*iT^re-Y2A(@A|&5-wFkUhAKprnbXZ@( zFH51Z;<8|NU+7o*ktTFxoE=`WqVXbb7!QCP`5Fb`z$TfT)vG>cJiPtB=((TF;jKAU z5@I)1(d>6*U6(JnEXGfecb-#}9$_lVm*05Tt#I&z=|!THX%B5s!-WV1?x~>aRYDg* zN}aVi-xufl(c!(>`B7q%Yh77gCEd4C;0;I(Y~w?iscQbUAVuzPav3imktYzWM+yqI ziyP`c)n2DdYeq#rEp$p5$F+}0$(>{iK`?y?xh&VSV9Ycg){l1sN6d80VQgS}x zHwQLx!f+wLubrcdrrUC1?V!CP`8`*2c>6Q-3m{DW{uIfBzXd@ouUo%>QEvDfy1Gpo zLksJs{9wZJ+HJL$IBM6KZ1lWCJagM>qyiLGh*m~bhq4N-`31wePG52)oXhBk+Q#SirXF(71iwjGam^&b zIx3h7;>Pd4>C)Fo-_0aVxBkrJ>>%wDy-e9BK~#vj72X0=~Bh5O?+)i>e-XHe6F zjsB9r6(NF8%kT)Oh!>VV^#$ASEM2He@g|{~V#T~8;FCHl?h}>^VKsy_zE|yfBPf(r zG#(G_d3QQo68W>%5gEVB{Em}%ngJ?=)Js`c7au_>=`9==7uOa6Rj!zr-yAmQkRitR z+eg@ModY!oM7@fv>}(wc;WX(8G;=x`rmVew{7Wf{;wrRQg^z(%sBM>CxE2s-QAE^)II}ww?KodsNv4z zDi5QzQ^FNu=vPJyp+N~J<;C3hE4N5yU9_w68(ut9N?~inu)%hZbcm0OxiOESfDG}L zfLACG0H8z?1kx?^zrhuiG@zyhVfv!^1*s>L+I>=y+^%eYZO(S98X` zB0gj(ZohT+(!h~UeWI2Sa{g(-$)A1lr87NOpogv1D&KoTQhak$LE&9r(WITy_iGMd zrl9>b34Jw6y3T8>Jx?#Aqg21yUz6oI6FvMwkhPoXbT6sfD#VQ_e)luc)VrU3N8=--eB?PKC$%@SE`vwIygzi$ zw8Rf5vvGYKgo`U~EA^0_J&m5hLfG%dq;C&HE1DlR1k-y2{zd7R9?1f-)u8z1 z9R$X3#2%HF3NMb;7%aMctIC?0((ICALIwjN5a8m~RbmmcRwvK7sY?qe>}iBV|dX z8O!beVqiv+s8tLbq0pJ{Qe2mx z#`HIM}+(ICXu4XMtzZ z6Gf3EI&5pNoPO@$i+`b?n;r@J`zs93QdETO{oOW6FY_4<)1KtEsJnuJhOdy*_0X_q zm7yVCoy^Fm>8h>BRiOIA6#>#rSIk0TeJZKtw4f3+$!Hds6D%$+-hKg$=Q5(U3s%-1 z#7^B`)vH6^)SvOpPiNKO=byJzHP8##uVPoyc!{I%7H)7|doPVX4;hoYi>*soYV~;v z(owmI3AKlcL$d<+#N=g?vurl)wrM5m-0sSXi=KTfVEAJA1e$GY%5t#aW$TS zee6?cgryD{{B+L4xJ}z-lYwbl@_%JEv+TG&K0fxMSE6oCEG~xR=v5d4r1wh1ndR-h ztZ!6N&>ZwhudSUj-SxcV1OQgx~d_E7MsP;P1nIo;+->JBrb`Ror{WT zR?=}I>6|m1p+`jL`L51NlHM!xRk>z9gWHeBN>}P*nyb5bz>{(6fLVqy`UX>kjy<3M zth#W;;M6Ew+1{)nhXZHl8`1*4d;3rddw^C;08T%q1y{tJ?&WgalPRs;v9k;RFzo5+ z4{3~L?8*L?ls~v(;>(4`bN;e*t136|v<{V!QXG0-<1%0ACP`2>SvMA)ReCTXY!zIP zq^^!2J!4v|7%p;&G2-+1pb4$_-rc||`EZeLp_orAM`+Z25zl;~W!g~IW+F>txuGU% zBx0ZxNflI+O4+dDQOSjUH-sE6XqRQLX8zXLK=dZ6_LVBK-Uk(pyfRs>wNduBA{tTjyTOhlEIXLXMz3Wka>~VSV$w$cUBHE%*RCS>WljAX!#}gS$m~P%nR7p8@sq3z8YPpL)#@ z`bP#I?(|R>otN?{#VHclK$@A;0&AdqkMrXT{S`scLmq`@wQxEi#7$Tz8a)g zvLxmzR}C7g5x3mfV1kr}>4(~|#P}@*Lyw%v@+tetXQ&S8tnR1bbbkU+E#~0+#CF4_ zBvqapGNz^%%)h*TVv*=`zC&`41M_E?{k^_CLK74eWSj==Jm%nX>lsrsFMZuBi1%V# z)UBHMnn85o9u<^>-xi1EZQ(saS#a-Tw_6QVRAiT0E8X?DM7wT$d2naISyUy?AI%{P zjA2Swc_GwxzMhtF>2xLR>ey<|-SJwWWwJ$3h1lNJ^Eaztcr2H7{x6vRLGXMRqc^lv z@PINHt&}e25Ca*6pqe}+0~rfnAiTd++&RdNt5o>J01rc z{a6T0JIqT?9VIojC&Rs)M zdyJ8{FcgJ}^t|5S{{H)`pb4U+F(jzozkwU$4}9#$RiOp713`-d+s3V@dipe;{sqvr z&I0}m6{%C0ZhMpd96d@2PktNQ+ykZNzP|H7&Tj&Xi>V7^dN_LAfFf`S+E-N5G6Ln) zoR>O8=`b18t}z?pBXmD6N<98HT>M7pr=yx-)75%T&6*!Gu&eHM!_KYCrMtTuB&C&>mM)Qy z?(RmqyGxPoRN`A(J?EZ#?)&@vL(lUZA6$Fywbq<-%rVAX-)+ez2Knz<*?&$kJ5a;B zTpR$b7I-^i)M_q%b$nghpyxIM`Zx&>IRVWf&OQf_Xv~c|V{Qv+#aeH4Tv>)4ed+WdkljW|1_GyCYrxdT*FPkM4X(MfQ126^}ppNa&wc@+`b^wa2QVGGUGQ# zW|2*Z@$D_Z0iG&CQi2HYqb7x&+c`3=lOYM)#VSaL-GRX@}J6WHnEn-Sf&8V@ruBl7;mdhA1d_=R^~`xc1ekpM1gP4b7fQL#lWJVQR1s#z*pI zxIYk@s{wBBordNr&N~ZkEsK+%DT|6!;DCRQ|Lsf69V{Q-wP&nmgMad3KyOmP1Yq!TojR(96>4dUGL>~BmsUn;hR~CFy?E^Aq zB{ed(Cl7(JE~3|pGGC7LeB z@_=6fUWk4p8Da*V?{Qs#!`&h1^6P`n(LHo@f9Y&tYikC`SwSL>0T$*(I9Vvbl*>k@ zTtPo~ARb^3NJ>uD0$cg-^)e3LX=LhUpui*xyM5Z|egdpb-SjgnM2 zdgsw$fU2s*1`)`gbYG_yxqJ_|s!@HtD_Pm!CU42-gctS00hgJTs?QFP$|By>-2-&r zlU1OK_x@z->~hH+92Ji=-Aj6Su`zLRnEKO>Yfa^s+hA9!&5pR*WV`;P=>M$v*5^Z1 z5-6Yjm9-Fa=h2mBUcx-EMc$eGx?L$MfsL^o8l!Qd@6qm4ax;M_+S|>iDm+dE&;3pH zV*XK%`Go5sAZqhn9};w18G?{bq@IHJ^!yg_Hd*bp#4+`izuus4$``5kIiV{*f3x)e zo0fsx2B?NX5My4ANelph-<*c?wu|5^iJNhszvsb z5e36r+z5U?n!ObZ7_+zI;>Y)BpX++{Dt-L|0VKFM9;K^M*RF?vji`FAfNkwnQ!Sp5 z;14t-0CHJ19CVFVtS&56{KMlF=K^=Q+u05lq!TF9l}|t;fvf)n2-@0w&V6wQo4lvJ z%IUYwDLHq)XvA1cC6n&h?w4p^m`8c8836p@1TB?PURfkI8%Qz5B^X)&2j2BCC4Y#5 zZx&zfdQMqb3-EWwYN?5R|C|-W=KvW2|2+^n!$Z`e_zj+huOTloe=b|>#krqm)@@bO z%?6<+4ILIOHB}$mb!pQ9(rf0x+oKuJXgsChh)BOxyrVxq$|?~cMp2JtRza8c7wfMe zNx4wy^fPN{*f;B+MCz5j{e)^+&t+8RC##dRC9~n07Tm=%LjvidQW{Ax8rTRps(zTD zPb%h>Sl(!xvZ+S`hAODh^=9_GC?A^eZ%tJGa)}&2Gf!onS-#GwHWlfVlh}qjFn`Q6?m~T?g?*K;(FP??Hp@ zPo|ouBqF`Upu4K@+@@Bk2ur#Jt?|DFS^6>#f4#>^Cl$CWc42@KduW{bZ0GN95DV?uH#Q(pXm zd`|u=igITbr93=Fo^f*FI;WPsu>`bIA|-y$**Euxq>1RX^+oV#7@nbi|8Q(~l}gum zF)1KTN!N?7<$4($o$xJJL1m@eZ^`y+n@vRD&M>ewQX=9aYktVG$Ev-<2-f=GV15~ei=Gxd(gxHy0Kid0r((tD?y;B8d0*)`BC1zsvJG@)kw>Q3=pC`<0IcyL;c@OSu-_R3~Uy}xOd<~bCwXUH`{_{fUf@i7; z0FT5vf3+p%7$Nw-=#IqT#D}%D=YM+AIG%2y^f^w}#U&<)wG1891mRs6E#(4Nh4?xj zTwuUhM5G`QDtTj(SlZvpO?+T#kd7|{&RXn}45g=Wg?K_xe$>!&)V-^FAIg*|PxYzB z6O)S95xjw}9C%t$D3iWHg58q~L`r!-J#2RGX!)G%*X2nCxYuV0D{X!#8o14oLTN3) z)Xuy=8dPF!rV~%TTK@#7>g#F`wBnCzQqhr)3C}2y_pd8SMR6z4N?GxPOV+-=2?;^R zhwg4RL-_}z(qE~+dEam(BzZ~+szzh9Wq648O7gMfq9MHC^9G(xpFsJZ{hD*t#rTf& zdXbowhf(=4W6Z~n&!bV(5hv&k_ertF^oaC@veTo6V8kCPR|bJS1pSgrGCV_o=4&se z;^5#Ahw8{YifZlbga^klp5Ki_NJvOBks8C)>)YBn2%wL_h(x z&z?1JC1aKf%e}p|l&=k4IUm^mJ~Kx%T)&2Tvd0B?n#hw}TY>gc4T&qV5kaZ4b6{Oo z(-BrfhM{9J3T0(=IUs&`4hx3|H|1LF`J-<$tA8#n{RjYSrsN=9-8uEH9~xgz!3UlN#cB~ad*Ct88QRJYTxScXa`^_1*BMm+< z7=uhW(%hOUnC01#)7Udd^6Z*)8N*B-QqVqy1%^2TGIe1Yg~Y&Hh`Oqs%>ji3MWc3& z{+r+fye5oajQ#&=(5CP2mjgNlolcYH4jb4yLW;&@K8<3Snr%NeZ@0$rIT#irW;u@F z40A~aQ+_OzilIJr?|tjb2$qI~fZ|Jt99a+O_P04{lOsT;BBRg=`l+JgnETazAam}J=oeQN zHbho0M89>jcq~mmq0>nh^OEVXiNI3T^A1sE6(vKI#NqH$;eyu>(`YSdig(z;OtjV1 z;yo@^^VfSbIW~3#uuId%hFMQIYPc$zSt)vW4m^HSIc*E5A!VPTS^t8)o==1|Aqb>x z2b7scpkxMx%KOB3sL>y1(H*sLA4ELYQpj>n(TJnmIO}#^ePGr~cD0I8AmI9f@$Bmb ztjpTq2xR9|K* z%^i{Qkppxl+xlvKHnAU?Km7M5kgB)91P%h{vFr4zB`w_vvKFPkCQYHaZ|Boan%ESzW{8g)Tt}u+h9A55dW4#KX}JsO8M|PmEgWgb^9QHONfr& zH{R%2pO^c$Ir#QYCBma^Cd)gzkEP{bj4oOPNNhc(E91k9t(F*&Su$_WN)0=omKL}b~%Td~DsYW{@6-r;Q? z8^&pjsOuC$xCe6U;Vtv`IDOnl4L{++lBWb7@9x%jEtg(!0c}9onWnHI89>T(*WQ1{ z)3LFV5-PRG$tpsnt=*1{u7oZ|q0dET6Ye@wgMZ!Br(5mDlnE)*>>`G{{}y)woyvSi zM-Yw;Pm$u2$mHaZ)m43%7BFFAc6hA7rrfV8y46h z1N|lO#u~zxYWWe#ji6IjX8avLI#pEvC_r5D1{3K11Cbg@nbXEF_jn0dayM=hgnwU? zh9e?6g45XMl|(w8mk%L}YgAI6`(SK?pX3+voA9 zW9-X}397x%#Su!1dHA>MrMdC50jZfr%3db{%o_Z z$pdpinFV(r7!|(oV`)}8#y!*CAbbtL*X(qO7SrkO0ykcxGR;_ZVUfvq>>>|HwtpRG zH6r_S%>w-yZ%u|S$bb#djP*gU`~q;#I;BBEZXF38LEVwG!W(Ik?h@k2pSm4o)l<~QQS;q!Cgfl%2E5WA-=VEiX=nYYjvZ~LJ>9lNn@FypQz?87)^$C{AjNe*c%?m zJ6Mif<<08v@SpQH&9fjKdmesq<~3dw|FWM`Sj=X({wI!2wm%{ARpCGCriSr0>WdA6 z6265ewHX%ev1xf_e5Q)8a(a&_-lEUC;({c9*la6JX)3K~r#G+lLv&$I4=5XpYeWsCr&+AM{bS_bB;S)(wmCc_R}7u1(5;g>E+sk`=mk=c6<` z@rUxhm*^d&*Lce*gx?QmtzO0x9P8YzskS_|xAJ%xrl}gV@3iPw+w8KIPp?qIxGSbO zapdvtcV;%rnQxAXfQSeOBd?$!3N^8;V*nZn@ZM$65(4FxqS0evEd_M&YQ?QAVT10= zYlwl<4#*h7S}O2LMp))YW0Wy7Z7){PbNlgnuN=m;2H%F3$K9Y+5Zx!$q8`LIX-(d- zJZuSEL^v$w*u{rZT173n0Hfn`WNUQo&vF)}px{tc87KU9I?D^pQGhZletw(&%Gn%w zG0_uU;itQA*rNZ_OyZ2maZ2v*Hgt2>6xtUP_^H+;ekr*+=@mcHgIIxt3i?&Xr)-C? zq6C2Az>pB%#zy|=SD&XhE8O^eBE;j*8is1Yj_Q254Juq~E(=+zC}9N+zby$9=xG#m zPynLhaOwzQ#Rkt!x_491wDYUJQZYO(oC@Z?a1iQ8Atq{)8Nr1&irL_|VOBH`)pp2eFrB*5W70BwPP#<^0Ezyo)j_cqPfcnkH;U4Y%! zDV@Clpn32_H<;;wVfS}kthbHl-3IqmP9Xx;NDl#=o6Of5To(@SGHf1SpvTXcw##Hs ztWF~u$f~3_hw6-bdHP9oX|MlyUGSw`;`R$;ebkBmIeKts8{(s&s0f(dM*nVykc9_# zy$?)vIzrKehXg50N@ECj9sNx6*%g)C2z$7_sA*YYbK^LK3(Xdg2Vu@>I{c-DL7chk zda@iKmGjc7(jNqxf1LS5rzr}}+SlKty4Z=(fXiX2)2=#Tz+%80EV}Kl#HeU9i@7@B zK)T2F08HV~Lb?ja!*?qEBB1I|N{*{{L@ML-1a23byl5-VUb?Fxg+mF!egr>eIbE4~ zYf-!Kp1}dz=g6`;b`aSgDzG8Jw#exPFH-a9w@fqLb#UO^vZz*e%SoI4@e26z1!fi~ ziT9`l@Hi^vue;ImP-V5p80(3pa)E@~Q3x(U)h&mTN`g)ha9xF#6TGgdVc5$<5ylLhHBvNVi6Jp*$dYs7UY6Z#*dqw36}G+YIPR9NpHFw2 zp6wg4Fr@(=ebg1@*)vrYmP1fR9jRS{2r2{dUUk3Gt@-uGT3sWPXd|srkm+$hMJtLR zO8I7_B$;7942FoAfWiNtK*n*e$Gik&dZ^?=ArkqyT_esfJ~fqzGcMW>52nVkV3YB>IN3BNk`pDNw)BK|gz(wRY9hhPNPN zO)VP!wl$ELI1HS?sPTsu-4Otg)SfYWVDxa=MfuKa26CGTci<1NbtMK;(wgsh>gXyJx@m*{Xpk7!8n1N&tSw=JmiS;4# z|1dBYIPOiDvzL{X*VLS_=C zB_u(Xohgr0cZveen%>4JFAPgUiG8h_aG^pPT}w~_JG22nTU{_;q1xOt(1Z~r&B9>u z2hQ8(>2y8GfAL5EPMeE}d;!ZEI3RUFNHVyjGxG9Y*=VN8>VkoHE=N6&nE`+wS(cgg zXNyR#eN{P62V}WM)ee4?T!|=IWs+WbZh=_AC19`ikvJ_drf}QVJ`=QAK^$o*{Q3sdwoQ8j_i&` z;yqg7U?V5@F9f=VmMI;3Xz@~?-6r`_erzQEnBPkDl-ul9pB zfg1gF0VTGAqbO;z=1$0uMDi@$c4#7IdB) zpb-YV(()M(PO<~aOj>TQj2B9id6&O3KYQb`=_1$1C)Mi&^$w7j`#wilg?^(Wkn4UbQ) zc54WKl2hL3G|*W@MP&SV>yQB{{@o5yOe?0E;v^nolU!}Q^mVNdj#oPd@Hga5y?RiE zy0erVmL424AOo}fRX?DKnwN^<8Hs0W(SY_45zDj67k>7aytlXWZAw7+S7F{%-pJ|d zD=7%w-$(@T#0hpu!-krF5jXzwtuez#YXgDCaUY`Rj9YT+~lYseqJJ`25phNp)>yp<$21{~VBYVrV{2I~WfE;3jBD z)iMwQTaPs0K2_E|+nIs!-kqy9=m^Bhl8jRSD%Fm#lrBr~K6rc5#23yzS2tuR3 zV~h6%)7^mW{kB#j7IJXK&ws%Ab;TDb)hN1xWATlJ*8wF2y5djneMI%+cUCOD`#+!J z7_BXU+>>C*>z047_B+|t)$!1POwAxj0uHu|n^%FMZEN+Cr$sty?&oQW2iX1?lv$3e z<(oN!Be$@Yafr?D#Kvi$V^Zx^gZgvOc(YTx-r&K_Ubn9d zuP*(y`y1!(Ib2UoLA!?IaZ3A+X!B;L`@(%Lo?A1({i8B+gk&qgvNkkxL5-_~n-pIf zzuGh`GzFE}M`+l}8$*a{;Z0L@_p5w+P|*`X@xSjDQC+LADzY&z!7ku2_M^6j@ zfyK!;lQ;t}eI8(0>%H~94#ct(=eAFWksG{aILQk;QpuF;*?Cn2Ju_ZwAka$d%-#XR zA-uv>fI%xY1Of?XQw?W`wIA~&ApBoz-`(AnQ&9mDBqX<=lh>H(RaJQC=;$To<-%50 zg+VNtvHHJ5alH0yZ=9S1mmnh^%MJJ4P*2Is5W2Y|tuChTU8k<4MS{ZnKMXM+9F7ZF0p$FQsz>HJ#kG4<;uEYKp-%WsJJ^Ks9IOX$)6FITrN7d(^^z zXSUqVLJ;-?e0k9q!QT^?iZ=V%dHye#EHkjhKYgOmY;eIc8Tu${XlR(f{*m7ePpi&B z5H!kPzkV$lP5E~WIC;WA(%HiZHu4Ui&&c-Yfe7(SpHbq8$@~$EL8|*IlN+60NvtT; z>1`std1heo>so`wZ4gnww(m`J98FRxOv^$S(sh?_+U|goK`d)4Pb#?~7>j?}DIF(rw z_TcVEgV~rFW2xo=?U26J`80eIcb?78Ee43GE(o2*Gk}P*DWU z@pW@sZ5Egh6T9I>m9rBZC_<;BFZcqwQ7%7qEDm;)cq1AZ5s4}%B?%P_EWe%sukXO0 z{_f%7ChKdphrCSba|*W3$mSzguZ_rVxXbSX<38O*aCw7%EjS4XPaOLazx;*`#?RtG z_Ze6r9dzKxw2Y160OP=e^K+Y#2FMQ{7bB$cNC0}rdd4c%*vb#$V(2;7*A)+uUDvm%_X-XwX2 zqA^MF+NoTuc_ams^AYOeTZTu25$e$GlYepV7w*jp3y=F_=umZbWdHD4fvKVxp z7aWl+<|H7SjqmG>i+q!H_whMyg?WrFXsjS~1Y~rz=uXe}NF_h4jQ)S8$64)cC#bb( z4Cv1u^;YaI|GNWSafv_)V>w%a1Qhn~USzxl*a1*tf@#UdpP+rzsVXo34sdu@!v$_y zfTd;#a6;%dWC62N(bv9&vO4#U-}{rIUM62}nrTW)j|V$8Tq|8*)XG37`tque;G$Qh z=4P%wRPuesajJi6eYs1}0+gorpRz-M2+#oOf(z-WyibE|skH~sFm>+XQ zFF(tJri;%l90@7uA+QY5TklU;myEK0_a||9_SwW~?c|nhGMM{^X!`g0O0=au@g5sLs5y&JSY+f?tm5Z#_5wr!a5EuzRTn+#9sTh?Q2#72 zR<9{@+yGHF3|MIk=Ucf;Q0FSXpR=w<+i&{G035fB*HJf(|3uyOj4^ZDlj{-L812zu zF@a-5Fia&dI>w0CmVsE|*~Lr7^)1uzHI&r1ZHvm81t*Zy&aBAa++quA_JaIJ%xU%; zUSn2aBT1vbKb5s5On~-JBYALi|B`rj~`Uy{yuSDbCy zcDemAW!y{A-Yp_1H?Ni#&f{VKaQ3Sa%dUuniSm&d0-3$z4!CvPb4w9QsZ>pA{1j%<&cBB)TWcoE( zK$Y>QvVtm6`-Usr4S`3f*DnHEC)!tH_aw5iC(I+{9AI98Pz1`sKBa`hL>rs6kd5V+ zP)7pXjCwX1Oj7AoYyZCs>wL_yGGAtL0_Gf)270=F28CYJtrfSgir35W;M1-9-l4_( z4}R1tK+6hI5(bpmY;>&-Foi9d+X;#O4O{f9hZ~cOm~dbG(AmyNN;bM(nKC6uL)MK#fb`Hp~t1?FL%%hTxfNYa>AcQC%9_Z;Sf2q|F`VanbS0z zjJ+S7Dl9+wgk(tax$$9|@<7Zh!@v{buz2(_OdMBUK+2bl{`}^Dj`%+oh9$N>8*Xbh zCEpl-JB&DdSQQ6oMn{c5yExpN^is7gH7y^+@_WRI6Kb^r4CYHTd*+0x9S1ErB1fuN z?^_a6c6vAMkdVW6?+aUzE_gv}YF1ZmLru`g{p<4VTY^uKp3A!=ZCt|y_!HF3>v}jZ zqLzsM?=;RIt&Kx}smbHwTbvFI3&8qK#i#Q(8f0>kUouh3p?TuuH- z>H;&emiazB`}hezinlIph`1i`*xrLXsE@HwWdBWShPM4KVly^U$^P|Y%=PcI>e|rz z?N<^8rdwPM{BY2-=vj*fuLE-tVCo(Q@E`PMOqHm0u`hB;8X;eInjs)Lo6*`|9k^0n zJwG@715n&^2*>jv-j4mB;8r3yjQ~8PC_qX9mJI^~hQ0u+V$L^>(%J10Lk}9tV_vCl z(?2qTg@Q-d0y%7yT~_~rHK{&Ece2VQypd$Yd$XT?@IGO<1J!k_p8dkgrc8V12CI?z z_SxZ$fB}dG275#RlIsS;u|PD4tawa(`a~*t-En8jujazG0~OsbrE$JP?E^C)&I0b$ z3jkwLK-{8nrNYGhZ}Ui{w@Y+8%M0Z4K}H|tdj6fD>iLwQxq#y901U=y1%nU3toQ+r zd!Qv|78> zH*PjY9~lqlCf^CU*speFb|dBvqjw?FW{^(*+5vXhi+-ZKT=|vhB1xz=?hF}Se+*M% zACYlhf=afa4522nuC3376Zf7YBe{1uhMrW4|f(;)9q;`*52 zU6vROz7)`5f?Eb^UM$?{YE$0RO_$X8B5w|~T2MIs3l@W{42||C^gnNHZ5<2?V+dxC ztpFeJw<#%F<@R9c(cMC26yS~Rel}y`YGMT@bGQRaWl43ln357MBo53NdQ4>=Zs-(t zu*Kd(q3uE==xbTf^-I|xQBC^7%A?QolZ?oC{Iadg{H38lL!&znm|PVQw$K0$DHyHj z#lRiHxhK=|Ugm7&sz=DvbO0IfB>-Ihwtn?l*H34oway+yEb?nAcTFmFw&oW~@8LH4 z4o<7)n9uPU-RnIN&{J;VUn$l8)m$=no2Fn(2Sq+{woK}m=9^PSRAP<-QG4Lgh5}Oq zrri=-(STVq24D^+jJHz0GNH#aEIGGtb6);}1qZ`A!)6*zkCffm8r&W`pdF0_^G;IcMF$Sgw9r8ssootrN(ACOTySxO>C`+m_`+7vRk;!9nV++!w2C0I;}d<*96h{O+3h*Ow3H3745g<9fL{r%>{)mL7?fC+U@nqKIoQWas@@19?o zv?V6dTOg7Z#^AqOsU8Z(3$-pRG>RSr*(R~)60nDQR9w%atfLd&-!BPy)|d}Hz+@UccbZBNrRMqtKa6aGBQ4PHN12?SXqc;vRIzlz~jP` z4{DgH1vB!@E^n8CUSu_9N%-(4@!7P?s>j!u6b|7(4dls;%iFeeV`$zpm**ANBkAP8 z`n|gR&g0&`;*=P>@~rfvB#jJv-$9$f)9-|f{S)n)K{<|3o;>N0PFsj_CP5AA$Kwgj zqf~xvPK;bWMyYb(y_MBZG)A{O)J#Z--$L}WJw;QWJH zVxLwGS1Ug$`y5519gr|-0BC1k3$AW$yHHFvL!Pq(J+;|4I$1~r@#nnZBdkbU`GotC zrOyIGP6dtB=%sg>0@vza%c@U1l#6G>jmq4oycWSq*0dV2x5d;CfNvce>Iq4Rl0~nw z4n%9hDQ$HI+!I0YY{_*M=q5)va{1Ob0lcD0X47x>6n6N|Y3E7a4G(xx)gej><(PPh zKV)HbhyPvB$AW|uwnCl$=Cs%=t*8i{+~Wib5b{7VJu*_dat_ATN@-^+YX8*hI)%Pv zn6&fpab}U{>SVi5zGa}VE?STT$5*oCx+cMUpID6e+aB{hbTJat__hK&&su?hQ+*nt zeHC||dy!=l_sBSbg&cfNOfcY?jO@8-HjHFlHO8m{vMn&7mEIY6`(oi_#%JQ&lO2uJ zw;tQ9n7|Mf_HYIGq7YO$UCB^yufhtX{$zTv^(=?|3E*$Mg>LxBMk3tacg7}hwDzTp zYs5Q!7YKuAiyK&g$^l5O_ATPToF_4*I4%fx3N!Md?tl8CySTScRlNbx-vA^`8*|l+ zA6>U{m4})=HHHBo4-Oz;#X;K_9NCtK1>`%8LUJ0(6PWq*y&~WIGHf@;#X9=w=9;_C z+Rx|Eh20hcUQUupcUm}yh?K-7xdZ_x)(0`KFlWXvBKfanuT4oI)DECv-9t^6XFq`T z>MFS4x!kRSpHcJZK6mH3!*ldzW)?U(Zjjv%-xEBH=U$dDyuT;*D>Z%9@-465_`Av^ zo*tO7TXH_{3`PbDf-wtF7Ipr~uB2Kn{^pNB02`Lr_YUO7*b8O7bf9_vmIVc z&h$YDYu8QByFBHEuOvp5$n{ex?O^%=CiY4Y_^QA9#a&s|hmZRn#PnkBaT2U}`Y<;x z-T4~*fD$z50prLvcSo`O*kb#sEzk^*3|`=ZL5{%9J{XW-H?UX;rJb$}e;X-Ol45Jw z{}h*x0w2!yV7XO!;0-XgJ6KP$;ke>{@Zc^tH#e{#MFb{90&F}cpieYiW`u%(M&t{| zVQhpVr7(6(6rNwmK(G_waV~Qp#6DM&3#10U@QOtL_;KJz4BB?JKEWy!u)&NU*6#3?@^wddh zOz7rUR+z}eL+FmhYHgH?ehAc4+Ms#~33B}H6@*6YYt;=9|5Do(p^G0sIA+BAdwYcd zwF*Ge0bt4!1Y&su|2&va1qG-8zG`pvL0sA04Lvz=0AeTt$0p1_4Y5A?4|Q-RO-Wb=`BNo8UAiPK^>Jn#W->)zY zlVpHe*aVp5Pzzi9v!>ECp5l|>h~#5Vs6qT=6)&ff-hxC2cViK*lr#%!V&7TFsGW_@ z@tge#Dr@GFKQkPl9{ea9TOsL3umYMID^0-4;t}?*HF&?H{*@t-dsd1@SxL$v_`-jS57tO)@Ku)lxG=K zDlY)5`{nl~dh4{zBTqr47&6M&lCRe&h#SYub(0~*jQ^(Tn7%QqM z9lD6yiC8k_EbO)pL`lRYyEYBp44Wl+4FV&SjY&>S&|@9`>GRtL&Oa?&&(Z8CJ&6?eXr3}%D}(1U$jKsog6VdH~RG?b+x1E<|OlpZ+()+EMF&_Gzy(d?=xa%V!1Yfb^d4A8cI?!q(lsC0_t{0&03MmHgwN z17d$}9H%N*vdHQG+B<+P`PH}<1q&-O=3rN#{ga@o%Yh6B@cJ3fX}ZvFe>R@Wj#g?# zyUAr=im7$SL!>_h&@6%A8nE7nE1hA0dyAcm8v?rJv+%b0_eq1iV9K%m%S+qWrNT>* zq|mJj5_jX?M9G&=BGWTuylQ%>F_(6mb7IyhAvy;Oj(WwN5^1eag9NWt)%DZ}39Auq z-TPp(3uY8@di;?g-G)ECw;5{(4;ph<(I1RPC#P1r=Y-RWXvc!Pnktac7qjLV%IV{e z*Ldib95op^EW}ZtxtW%|-0S*9sd3l_JY9xNJUsw}e>gMFXp$bgdH_RO?@9@}>C8UD zdL~0d=IZrd%yV+6)?Sq}S>S?cD1wM%x7FAxm>()v!Hw}Zcdn+B=I;;tt*+z;|JPmZ zEPND#Lna={#K+($QPLL^_-L!?W5&IDU<-KF&4IdnOzE62UIYj}ia~B72mJOouh zZNghyFGj(02x}80n{kSKn=%jKsi4d4ozXVgLHGfT`SZ`)eiiwl z9KDMbC&HB(>y+s8$Pr-v6yW|px4ne6!{fX%nT<{!fdhEHjIKaHt&$je`r?XLLN2$! z;!It}k9xB42Ta=)pUtHwgjT5vXDO)Ue*t6xV7V6tX4}yx8W=nl?*D`d0npidi<(NN z@Ya?7VD4umSVLDib_7@vsHQ*h0kqEoYCir=^ZF-}JA;gV_4e4SAq(Grn+J7)>_!=M zPm*5bca}^wx7JE@(tpGuT=vu2=X33w3oQd^gC1Ifn5Q&1`h%=BP$=gVncl$Nhgrt? z`mP30d;ad_E<7<9k5JEO1HOggy{}rD>;;}@Sj_Z}ZO_cr=xT z`+E=l1DPzf1bSc+Pm6mMd5IQ+LH8SV7SMT}y$-RHM4d}pGY0^5_3V|o*l@5k(`n6$;;}2>0PNph9Ne8!4 zBi$oA&IGAd4!JW)Tup1A1N+`rgE4Vjze8x=&P z(mQ>vK_1+o4GRqnfspXMMQP*yj#jFDFSiXC3d6&_0A0cX!wa`2qmT(j{-P;{~*Xl5hA+&c>t zR6Gy?0ze_eV$eY_qB8mIJg*OsS1?r`>HRPP*M+~ZTErq|{?o=lV9sZ|%_k0O3z^Sl zM&HdBa`Mh5*TFDLc$S3w3qKLk-Ikt%UbLNy2Dlcb#q6||qU;@bmcb-AE{_xJ`_^Y0 zoOw71A!t6%pMnWlA6rB#eDF>Fka=sc-?WxruqWW{wbwM`_P7A`&Pxi*h)`5`#pjsw zaN9+Nm9xs*mR_?wTX~i0=WS?DgwGaPgb)m|x(7aTZm+KweLqLUtTm2E`abye*t78m zGXD@)z-0JP)X&E6B$nM50vEaM5!k%sW_Bn{e8vn7(Xd5iWWUs3+=M8;pw`ri5PuLD zUxI4l>9UyDc(go4hk5fpH4Yh#r*UBFGEWJe01YehJJP+gwjCqdc*kAa`^HN8w_olU z?o6~;mN~qUDpwA-T3XkgtL6@rLF>wVgG6AbMiTwxCUGhi^&X+m$=l$_T(w}<2}H`} z(VsDFoB4L1mZ?c-F7C+(kpxAK&BjrM-V=Y_mh($D+dXlS1SDs2oNZq;L(8|d-|2Ex zRZkdMy)CUpN7k{{_h`N7;IbU--k)$35jjE0ReaZh6)8@F6%<=PcW@j=MKZ1GeN>LN zpsf)~l3-&Szj%8nC`q!^G zg={PvzMkT>`pzF&b<;2T+n`^=uG|ED5B5We#AJqloiK{zg#t0dsa|v70d;6#UD^TK z3)?NrabrtU8=U?ZisYBDBV#u93oY~gR;*nodCy&b_A^88Pi|>>t+?$ZKZ7Uq1eodS@(RFZ$~>R05*s+o{bSUBMZ%UR%4|jQo*JJK0CN+aOf#2 z4vtN0M~8+k3$d(hEuvbYWA}K0sF8to_@2r{e0bqZv9870Q`x6i;3JcbuHsQWJbQKF z=67x$@(WJ*>iOUug{JE@R=PA8S_1=fi$%2s@=P9>=eA5HX|=pod)i+c>;sgNR7CEK zlB><#?6=(vT9)emz_Fn&F39Sd>V5#oYTvI3FemE?8Tv6Jx%wkaMz=$_s(#X)+0Z?= z)3ypPlFR!>!0t%#Xh5Q+C5_VEX(lM!D5s1PBLOUSv_M$9Q(D!x2xTB8^7%VKJ)8)< zY8bF7v55IT+|otTe2k@}a_#^hvK=+D-p*;j4_;`C9lYj_6^X#_m~?m~9@ZSQTM~t{ zW1c_ZuMk3CkL1kD2);qXdv;E_m8jw7F7@LR&DFuQmF!bXopEnMU=U7tXPj3BJ=J~ zyhma>=%453V5vVh^+&Oad}WBIrm~Bte9cLR`JqeqZG<>UldlWjz&dBz<)&kP)xqHu zmENU215u>g6Pn^ytJ|B|tOQ?9NvW=<)r=Ky@b0jRJV0k1cP5yX?4B=fy$SWB=`PqA zVF%0`YKGu;uq<)$U~fh9)C67nP^;^#gJ~5OLo7Nj4G)5Z#7gkF{j&CI2F_)>#|eno z^BZ*uc6eKE;66XgE;bD8j`_dV7F+e>HOXbOnzyc*P1KV{rn6lBh6nuT#>AT{eSr9{_`LuKfFu^=OF_Bd!;jh-J6c55% z?@&f^xg#C&aqlm}@Ajl9TOIb@_y-0})tZWY%vQ5$dDq+=9`$)rB7xRW-u*UCgPEi0 zJ2?%jF!Uoll4cAA&sO9~rWL)T#Cj=J8LkI`Qx%iDChDVsV6mfk({962vhz5PoV)u1 z)x^e8@ZcLb^Kn!T;UoPN_D9W9{u*l|7Hdq(!fqiqC2x7m@2*HG9@>Zv_A4_uUkyrc z&&X-sWZpzjQi&Oa1$OB9cQ2GY7bo+97oxJez+Qc0&dN!B-i}x>>y8-G5xC^>EgkDk z));PO{LdXXQ%kFI^KjIYcA)JSad|#uY{>F!)z@Mkjwt9>+@WwuC~`UzqZt<9#WHyg zVoD}Qc}H~b-8Q=0*&V#Qk$tdZLx_DB*638Nn@@+1PpgrX448@SzRqkiaZIUKhF`5= z90U<zx->M&xToujgETSRCb;jpl8M`+c-7WyM3?PIOuhmY65 ziwA22c66J`cK_(QiE)tj8p4d^XZkBSojtTUS@&y`*pgcltis_5&wMQCQDg(0aR-4ATHhnDHzHJ?Sg%_Ry2Dh~qSh3Qtw$EIDouWW*`~4$P z?g{8h>N-#MD;=26*rw9ImeZQND}iywv6C+O({<&s1jrc?UC7dWexc}FxVoukkO!+*f^XNCTyeJ*U@8E}a+p}++#Q*GfqshxF zH$Ru}E!{hui{$yVJp=daCda%^ESmAos2^Nz`En=v{^Z>0qDQHCH|lA~;}xyPA`$q)6& z2*R=y(UMO!>*7Cm+E!<&IhKSOl&EOUA3=eR-Wpcu8i7|g^D8QN8kc@75SuV-iSh3B z7=R=B09{9jFK%Y|eQIg~a(Povp2vxS#wYY3`;i>JxkR6tn(gK#N7K{Nv6HK$6eim~Pjso65f>?e^01`u7Av*ekUN6 z5s#rG;^1Y(%WvoewA|2D|JR=RGQ*C~unJIHL+XT46upe%tzm7)4PyL|EE`oeO8E1^ zr0FydH)Tp~)y6S+$ZG*cHuoEW*CnQAlLJ>KOVGy81S!0R+j(n4@+YP38{K2@u3`qg z!@PF;I}^)Stl4>U?g*Z$QPTUbN?R8R!2q;fBx)2 z2d=Xfn>f9j4a0nn-<#Mq?{g{IP?4rjm2%X=UhdI{wVrdl)-D~RYBwHY)~{UeC_ZVr zVgC+UqIwhB_uh48xdFe7jsO{Jo%d`{Y+ZitmwY9Qr=&XkD`;uzNAn?oY;^VDVaKsi zL)l&Whcn7nJx_0gmseP|*rbgj*1gDg(y7Ew`luP?n=^d&XGxZke_!QR zqKSjzM%V@J?ulKDti7n*G7=i3uU~Pjf+P>*de2RAsxQeXn|wgpDfK=X-A{AZJMshl zzZL^jzQQ7sss<~3pkgnyu3o!Cpqv#L5I&vf{;9z zmn;c6LG!9O;+<&AVIz@ZESvTZLn@eOdAcFN&16qa-2L%wa@yLDo%OV5Q!z>7N7vlO zIZ|80qHezsCl$4{rv@w+(g&A3KG!mJG(so&g`Y?MTtx**&>-M|2Q0}Sxo1tjxVq|B zR1Nu|L-d&?#6ur3`evnwSe5+XvS{t%8d1Q>REOyi;r>$dee57#kO2h|rZ1L!mlO)R zE1t@NXS0cAH^S@y}QOt}3%3kLZ4U@8XhIHJ@OJ5kKf`=hPKZq<}FF%&ER0 zp<=eA#eQXkErNk8^ipr(;gA#=sb6iM0`r5;Ym-%0;k^MWRatpA60lsZG_~Mz_5iD< z0Li@P@?S%a%Te$t`UMtYsdt_kmOS{)r`c*4dB<=B3D zR%vG*e|Hy(&aEe{zRz_6WxDpNePGAd%ShQrajRinSyV>ubZqdI(^p0u(mPX{M~jEK zEAjPIxG$-x=e+j8V5A5WvtcEQdP6vn<3uXdAz5V^gWyaU5mHg(CmyQd?Hkw=@e&uE z!0fga6)ULgPhzh3HsbGt+L)qs&f}I`r=4|J*8%#PIBRE2E{XNuM zG3?zC*$W9HO7N5vs*=EyKsEsFos0+{wzYc#D^h)_z&Tx!$Du-e|0MZyjBupR@6C;P zBUpO^eZJkKhuk7(!&;FCtEbOSue!&iZcSz{ba?Gl9E(eY0~oq_X%GSFkdO}P?i3Lu1*BV~yOB=m*wT%3 z?{97WopZh4J?CG29(~_;uQlhGV~#PWWN?^hI^xFpetS!XD&5%~X(Olsnf4YP28kRk zBAETN%Iqrp(6Hq?7@yO)$R8$?w(FgY2mLi9;B4M(x}((Ku*h9~HJsRU_W-=Ys8(!Q zA$Wy?XB6eD+g8^NjTj^jXU(dh9BrFh&4(AuUsTS{UBtj@JPbdd-XNV6!Dbr+$n%M{?u6iDs_-yEfsbl17JPLy z+4gZAyb4M5P-!&DGKm~D2tGpT0bNLv8&6xe&hBt?bS^lLXN?tTi_|?^wmTOfCiV7p z*XpZDr$r9Yea#II4-jR-eO*-bm3+2opw(9)Fkx+fX4$?12@IysH8Km8Lpqos9b-RyjY{^_a1Ez#%z&1(bV;yxkbU(V8@7BNwu^bDtGeWj{@*_C?6ea{R>l%7Z7eOV*Y$%T8>RtYeiy~b zM{>q7J;dv`jIk1X#0ap~EH1D^=rb;~UqIWckG18bw5Y9=Mtqsju>4Z7T7qf6ndTe8 zwBHb&y-VF1|IQ^BUVW1|s%}Nbtiuh`L}INm*+acg81V^hOv^?=mjeDzgH=stXUs?~ zoU*~CRHO>}Fm(Zk|4=RmX!fy7i`+cvUvTN~3gyh%dA@IWZ7)b`5verf{Ou4i3f8^D zGYF>>44SRBiWRDc`Dupq*1ww;dPyqiP?qre!cHI!_Et9dRM<4KQ~Yhy=mj$Up@au+ zZ0_)O#8kZ)-t1(GR7KSlx%PtBv;LZg8Ow%28%UD)N-7B>V)X~|Cz2Rt zyJyDv(ovvD*>xwCLAftv{%y}6B-QB0aB6|xr5%ax9riy2g?ohwn!)qUr{)eWdWUJ; zmEs=|p)vU$_%WOrvd%mq1sV^@tBi@A1Wzw}3(F0Ad-Sfjax>%`~5STcHW*;t*+JVV40mg&rgJK; zDMEU#cXQ9Ipi*O(xfXkyx|xar?MbbvzBD!Xkq^}#W|0S1rFDHQtx(pS9^I{d7LOgb z{>Y%~rvV1W>DO(+M7pgzannA2y`peR3RKO^6h~G~!LOjvP_835%LAfs+ak*xB7>6{ zK#(G9a6%zu4hJh>g$tDWguxtid&Rk^ZDBWhKaT|zPtTitA-PiYaTOBg=s@PeSX9*R zRO%g0(s*)MVn>HKbv&Mm@$1NWWFusbCLOY}+5ZqL3K)mj^vtpfzFd*NvJWJ>yZ&(L z&@z~Taju*Fh6gfE5s&i@UQ%i9_&wpAc5Fl;#Kugv^>T(Hb)ZFC!RA3I110!>;2-R4 zM49ksxZMwsHZN|_Ju`#8N-&NFV($v;h6*oS=d}NN(~iCzEfmXelUkwlU7$*&~H`SCFESOFg;W6Z`>lj^m# zsvSN8>!m5#w^dNbTa-di-LblH+x?F?+PAmQv?}*|nc9vPt-J5`vAc})0Dw#Wg_}u6 zfhr_64xM)0mWkgh_Kt#=ZtnW4ek5Eg=8jgjTrRg6iT+x4Vo*1yh!|!%0PdKK$CarI z`!!~GX9N#laIBM;b4JddYxJJ-qKe;qt&Ne(-t0g8l&7;n**-$|Eh^|Og!E+Dq0yb( zGlLTkHcYjT;z|(tx+gO(d!jjc)ghatL1kQ;biAI3D+|pYK2yol{-kn5%+d-?cl=A- z#?v7CF6@Ib(}t~K-o70n5aLjp^0e}cT$9^u7vG@p3`n<9=Nzv9+Fx)3iZxyDyqOXNn`8wgI(df zWqse8JMYoA@-TdG-8k`riXm2b*&*aIjwjcm?4pm=n1;P^r54~%F0*&cgrjZ3x`x+C zrXf0#_$VJH%-}rCBVJ2oHBSUT{qjv}l~C+o1gdcC21h>h`sUTJB4Nz`mM6w^-bW@a zZ;v?gBR3(-NaP&*fr@|a=iLJeLBfp9BPd1ksF&Bt2#vCXm2U7AS@O0%SEZz88Xkse zp%LQK1W9I1I8zc3+0^^{r3$egygaZVF(ewQR$8H_e~+I zTW*R+r^L%D&0W}Fy&I?#Vnu`s#(o02Xz#O@p8wAKbU1IaGnzhZVP13mq?r&9U;kk1 z-^7<1z}B*Vr)slFos}1xf#9|v_cS8Y#uw7YvltA2gpiY=`d3k^)PnK$y%u;<2ukNc zZNl8;+Z32SCD&@f{QEu9j2;lG0|8oun!T?Cg3fi=`_`gX6SrgKA`NxUc(=AEYVble zM2F2;j`J<7Pfd+TK)-KNOYrLvE6VfrW)IR;wdLL1Dh75sGWex&l45N(mU`Q*NfmbI zj(~wwn*yhH1Y_P@I6&7R17&7H$Fpq^KG)^r&)UCkuS%Kx{Qk<&wSVpiwfzFrZ?*@E zKJZJg6Uc*+KPd>zsgz#Nzu)LIpc`0Dym={)Cb#;`H4tyJ4BbGHcV%Dp)Z3=Q;)p6S ziU7~+&3f#a;=D({VFmJ&yiAWY^q=aJFHl6sO;#n zcIt?j{ytC7BmPb5*ZdQpT8CdEhv(y{)M7q^DFx(kY;gJ?sGx+Ci^HsiA|ijd6GVAg zVU5$-ol-hCJLY4nTO@N{eR5IGx303lJJ2v1Q)S^2OqtHQ1qeD*Z(Ordf7a#kRC77P zL!-06wlZ_^{w8)*hzyFv!vt=~3c+#L!xYx}h~LP$NY=7SzHzIZ`Ss*k4l9b;-5_de+*Y8Sav7Cox^ZJIzC4s}wK3deY{UTSZ#|K6rkuQ@3zVMU!MpBAxd{I_T$P%VIfF;$dmY|_ri@%T(cbJ1 zll3Tnl+|9|tS3(DGLX6y^NXNXSS&b%U18FyCC%yG)Wr?cH{L2I_YxZ)Odb-P_7A8} zmE2iUc%|sCxRZ{13a5PGTno56RTmK;U~8b!f&N@X2mrz5M@v~-*N@xR|8q%q8T}Kd zG}E|GTf>?wVbS0Bh{hg#EHrKIaSP6$1?yrRp%ug$yWIIGi(>XX?9Zt;Vz>xvdBCH& zl@w2ecv4`Nc{<_V+di33+g{D;_*NKfI5Po(F6Y{hVubD}g3xQ}Fv#(w4mVKUoepDu zf8@fFFqG&QG;S7jWics#+-f1E|7T>hDeg#&DtP-7sgPgYmQ_Z)DzHyGOy4qym+w{o zyEWZ$zUo(%!IxM1eX1_>LQ{)-{~WF9!C}+3)`7sOOKaS36Z0JHd578jXtqGOevX(hTqILKQiQ#t&c=MaMN@IE3_w` zKah(|j+WRr2d#oUo8>FY|HD3CCcH1+|Ll6xr(vzEYSXp!#M@~kL`2Vq=eXVzG>*9ETn49r6y^njsHPY9 z9>h9~^AcC?YbR@m*U#!T%GD}%nM_~1H z1qx8oAker7kY!IrNU+ZyHQ8(LFfi-U~8S5jWd*%Vix4j*OZ zS|bdvmArO+#(vD4gkoot6D1?I0NjKTv!UI z_BekNcMlHb|KrC}ZCY0)D!xrt*FHbbmV0^Y4f?xgcat;P!d~@zo{I8TR%unrqF)#t zjk(bGFUlU1h0c+~uZ&{?og{utx^k$3o@oYc#o_z2XQ^c2wUgO~5-K6{-A`9u+;b%E zVb+R>@*lQ=>Wrqi=oXu|_SgD}VK3Ky9=u-OG4)K9{ zu#o;2>AmoMhyT;8-Aw$)Q+O%~m7F?Wd4_U%puIj`{rj;Rxz)5sB8nH+mX_9}5*0qa9f}M& z>P`WCzr~D1PmLn`yP5^lys8UUcr!R#D_C_^+km@zdd@}1R%jHYAR$gG$}fa#p(1w| z*Z)sLebjrx*h*rdySdr|>mEOH`<}-vAM&A6RL`&r`d8bb*+SuOL)5W8ir&?J75sUM zPzOLUxSPpoXSJVBJSyMM^&p$ATXbi;KT2AWkgY#g)EdKL?7W%YGrbd=!Eb~G__W`B z;3TKvt!Dtpqzhp)l^?-zV!P2=+w;xE?O*thDgNVgeTsCW^s*piwXQqm4^@jQSKRk) zd(;DNB5PFd@ZCaw=&k~~@Hg16q&KdPBaZc_Pgb761N?FxHfU5&m9O1DE$V~(;r!Yg z>@MCz6FCK_pT*CWEYsTaf>!_;dfX(1gQTSM(-s6@e|DYtAJvo@oRgX{v67Bz`%lr- z<|17@Y>Q!0yAnt1l9m1un+r%f(OKNayr0TUdw(oT`5kl`TLv6 zn&oBZ);ujGN-poNu~2&2)LoZD%+i<uv3_MqD96cYg#>!rhM?`=ffDTl7J>F{o6$BK z<_;`BjYx?KUf(af5TgGNp(Oyb`5k8bRdbmIh2u#&f#{7h_`Vf_43P94k+5KKoym-w zB)j%sgN@H;(gUT_Lg3G-6Hga665+4?!UzPFyh5wZQ=RN*LEn$!IE{XBqM*1x2H;wB znSJOVp%nb4XaS*AQEB-FwWL_NT%@i~TW=%dnCY~KC|1C@_TGC2dc{tl;`%ER2i#MH zMHC6xe6u(Tecsu#=)DbO93*BY8&JH-pK+<(Lf^!l=B?(uSMLX(Dy+!;z9L)Lpz%#P z{SGa;xmU{R&6#`QcVZR!lw_Yifcrn&(g3WO=Uy_XWClq5!Es74@p@afXItDUqa44V z0`!&@+x}?Dp6q>i4GPjcpkCq1JS z)is^CtuC!L>}ZjbrxKfb!oWbEc`N@g!SH$&!s*;aOSz~WF#uBfwG||HJt=}@M=K%N zQ+(A3{I%`HC{e(ZtHF6Mc)&juxt!p` z<&HZtTU&_u1Oz9Vg!x1nS!*ulrH`wApk`&1t(_$>7Tab$VfhG@2yf?O+tYf z@Ere(cwO4ntgc@L&$qL2a#nb+j_t=#W2}GX!Lm`cWbslP_KMc;3#to8k1wd$x^p-x zls>4=@75CD;WSjCTw_&yV%6Y&BbvAGFo}HoAA z^@tbwD)w`dM)va5&_iJJt_X@=<+YHj6IGd?6T&5i$_XNva8Op`9)uz(VW2_G@A)aP zVVO8V#QVI)yk9phNb-Be3o@$ME3+PwflKgPS+xiRQ_EB|U9~k7`*|1=5T8GX9?T6v zl~`)zIZuiy32K4HV=u@ZAAGw4P|PP(+i#32gbOMK@I^!g{3GTuc2go_Jmx-;O8cR?(?>$ZRSk#M~F18uHjmb!b{zC>ZBfuj;8 zJ#vs__CdKN4E+1!E`QUtBdEm_PXte+S&C*pfWGe0l!n}w!D;py1#RE!q1%wIvEX_U zStx~Hx#Dh>Wo+5zj$Mie>~&(&w$LueyH#%}=&|Qc)-glatN{ygg)XS2Vlq(@^Ryh4 zVTL+VMXKG9aqZt1K9ro9si=Le^sAkr9vAzG<=hg~01mspXJx%)ydQkffvwvM;^0)H zggMro`xw6{t_!Dwm_o4Sf>icazuLtvA9qp`JiZJ6UH&3-hC8`mYx#X7+}3)Nf98z_ zqcfsfsLm+IeDqou69Rc)Y#|;Ht&I&L1MS&iFi;r>1wG>>qLzj%c*$SOqTT1dVw12v zraAgtDu%qjVJxjsd#Be&j_eyTcOd3CB@0sDv%94I1Wp%1Gu_Nkxj&6mg zJrd$pUY9SVQO<=t<)y1YdtQ!GaZu@r9deaNY9NE~kW`r?>`F&RiW7cAwgO(tWwR2! zzj3t>svDg}%1EF`P|2rcv#!J?Bc=u8nbt_X-{y~oV+Io-LNa3VDMueYmem7dCmt@K zn5^kw1e$4aqNJwJ5Z|)3DWO~<46G&(O7Je{9oXGvi`RJ1nxowDI{PniA_knC*>I~S z5+Hh>Xx#C5D*IPN_N98K%6G^gwPp;$&f?=6*DQ`GB5&>)DDA!rbwJaeuuR zN|hJb%{v>W&#xsUs;R0Jdj++!upCaRP&{EE^;w*$pPfd(qKA{ATJ@7Z)g_eZEh<%P70HLWZ}d z`8~Xl-~e4X?3ENVwNVJJOct{G!Njw@YKO$9ju zCl84MD(+e|LG6>XCS-ne#=!Wlp#8E~ycP=qKgI3wc0!Q1_-5y z5ZK9`b_esUwel>YeP>&x+-TtGj&3@h|HAlY?L@Z1%abicE1mezzJwh8wjH_{7PlsG zyZUTOn49}%!;Qn1pe+yB?FSR{OGdm775Se zfwS42>XZvDWB?$2|mMNeTh0?+;R@iIhQ!RXx1pVA%5X~^-W!SYU+tx z!)-e5HqZW8FlYm?q3Pa`N-H)G!R3PbDQJMQSrABh+s=O&N zqZId=I;n1%UyqbDC2IR&lSKm8OBAfFq;O#-%C#2rx0TOnnsZ|Efn3MKD+kIE*pA;U z2{c^H#FD$*hSXsiFe<;*1e={nlea(pi0)zG zot%4wmls2l+XPVP7}h>l7T}{X#QxNJ-HVfzxE^X5G0$PR=u|G0ejdzNg?4M=EcT7Vr}0Hq{s;)UrQuIA%on%`R=P zVxOmvhvHSy07xAYGM7|cO6ZV~Yj`;UDYt)&CwMl zQZ2oQNz?3bg?(Em=!zH~AU((s4-yGYG$+h8g3{K>y}r{>#!1y6*yI_@y%()kvCPr;v zONok#YHlrB?FjMk5P^@Vo0=N*hE7*mQYa*!ZJ#FklmFewjA`qQ&E_=>;VAslpRZ9d%F&^97xSRXOE<>4y`(`0&?;fGC zd34ZuI%bz|Fn+!D6m6g<KYb#wWy^z()z zQ*!bmID7uifd&IiVg1(Tg(H{Ccl_Agt!Q%YZ=Y;!Rq3j{l98<=OJ*I^U)#%T#RTN} z`o27w@x|?I4Y#Wuxy=_h?TJfdlTqU%ab%>_;9qH_cWjc)e`yietUkiY%!$e+nGd>N z#?UgXe`$}|xwd>&R8$T&vZ4)t{p{RDg*3j7Ce`yGP!zx6#l*t0>!xyn8_lUotj*0^ z4p;jOILTr@ehdOL2^qi#ZK>7roz!u_8B7xm2OB`>czEvG*Uptp-X(~2zw@FNv;r|n z#yI+}$Jecb^DFPzb6jyy?25Z$GoN||fd(~M%~CY~%}+LGyf)ARMezA-Fjf@ij{v?C zKCcIwBk?up#KxZSCX8pFLV<#8W)>w5(Md}iU+iu9{1w)-PpGvnv~YL>KTt{b zs?nvy&FIit8wh!_g^=jt@XLv1_jFHw4V){O;C!i>lTL>i0Dow|PA6{C0_+gfs?36bkqhLgb5|WIX!84xq z*&Yv*`RrxoRRLs(7zE_1-5SY&R6&lC-AF<=e_+mU z4sRcDWRyav?fY&+bT{hM=a_+d;)rrzLOwZ>W}4062D*Hnc&yO*NXg+C9lP$5OYVyb zg2eM_rsOl)qGoW_WgYzDTox@zK7l#zirJ-zh_7V= zZp33IWKMf~N`4P#q~xZat&>Kjn%bHuL`>1PckEB( z(&)Cg_nMA2*ci^xU07}_@9L8I@bM%4!-rk$zvIOF-gu0s$^HfHqyyO1iOO;1h{|y` zj&kY9>*4`E0e2Xu4@r{3qB1LgX8I*aBZ2A%KS_)xrnj!?DF=4ri4#$&H+ejIv#1_f z5dwl-{6a{gEgM?D_f*Nv$cbfgG7b3FCuh^2Ss?x; z*pK@7wJXstI_QNsD;sUeh(Gv&Y zilQdRuMZ^+!G!6Av(T`xuXSruzKT013HAm;+u1saVG7b!Z>)%_6|qjN3j#J<47Rhg zSa6GdEvDk5wt0EUM_69_*A}D7SU*NFVlftzjJXPuDo{d3*xK%7qxkX3=gnEG_o2=x{T`t3?h8$7|uI^)!Rzs{U;R1uL z!+H3^(NTF0dmrfN6+5uC#`wq&MzGvL8MI?BUh&;8`t_7Qx?UQhjd-^1-qXE6^2`#H zp1TD2Ng)Oq;WM+ZaB%W0dY`C|F{$yVNtuj2RpVRB%r91P^afg{JR2rS=S@Y8UEkMs zAqR>w0p8qgH-k6WM+$x_dt%xse=gGaV!6Zy?Zm{y%Wvic5i$&5s(+AK+Pj?g$82nD zyWY1ahgqcc=PyAnpDzL6`D|6xoELlp&gyHG7)!Rth(-`pB1BzprVeUjW||&)Ldu!$ z(x=9ldF`pQxuQ6C@Y>d!b^R8-Y7hhCbxD=cgV1QlhbUyjdW9+bSV}v;YBqP#kjl3y zsLDt%?@p6CfgCgETb)k$d{3cUvy>*Vlhtd8C|}<1!C6j)pqw-U0*46#T2MrHOUy5g zT_K1lGlgO$DK5vg71vZ#U3cjlZNA|uix};Tz6c1Y{tu}`4vqrbF`6AZ+c}%18I3JQ zJOLN;_%2oe?&k>$JFv$+GcOMV&|BxXSNp>K)F=qG*9ou-g@>vMXFT^JcfBCGVgAijxmtQ)7#Z>%j{EWghr`nFP{Z*+e=*2C~+W$O!bZmV(Co%F9fXXBEmN z12S5k{I6y#o}4_AkVxIb-f}dvE%=#09t^tm2425Hgqo`XiNl-;QVVqqN}&9Mf}ylO zj)}ZxL*L=Sd-v{f+AN4rKDI{zrN9(GW`uX|3X4uI!V(l>l9Q6^8rD0zudlr>J}d2< zfZMjvCd0O&L|gQ@3YUu&zmDybS{Oj^T<*r}L$_F7JZ2&}*kHNw!t2Pm2kKN&HbcxE z5swyHSzK5Alu?YhGdz3tG?ss z)AghG)DYS!?u1sHcZ<^MSJ34F&N)MttAcwyurthgy|Oo85rEjNGPHr6-zo`o)tJYHbPTSSL+Jg z>V{k5x1YG~4HYFK(-0@T1lf9swm^;%LAzu?Zi^yuW|1mw7=iT>6-0Ne0sXz*-oY`~apN-+)p%R#CVcu^u;U0_?akoRPTmm z#?w@x@anzS5rD_l;d}-P=h;vu!jEYs-Gz9tfco^*z@8ce^vIZy4NCYHwRAF+iw=FD z@H%w{5@{gygyTI>G<whFe|s^;Kw8nkYzvk!)Y&ws|SKbGIjJPDUKHm88P{+8zu~I!*$dOj&=ABWn_xhQD zCmW=6#v$i*&nU zE8pHXz*{Z!MKMr5vZuv>lugxbFSla@ zFJHQslHi0e{mIJP6kP`cU!^ZeN%wTTE>;EYexiVH>c;DEU`QeGwu4wx2NA(_w{r4e z$sg|(mG|+8?605U98wSC=olEXAW-F<6T^SJig~$FjWzhi#1|UY3|=rCkL6JygE$)| zg_G-I^G?``9!+kzF^It#LE%Gr&z$Hj`7`0xZhd(nZdtY=i7s66YjHrL+*2t~js>2p^<2Cvso0rh$s zf<^TPC{(A+B${DTOv;J}TEA_>c%xwXG?Qu$EvQB?1CeZ+E*^q>@?-Vl&Ye48LEg@A zlj|=-_#QLBg}clM?`)krSqvulf&F>>#$N{~?~%HD+LZq>ndapl^K}YdUQ-tGXL}2FiIgM1 z_iDgHPEL#C>EU>$I*gEoR2MN3hH|$SDObi7>_x65WM2j3^l+>;zu}9m0 zO+brYS8QQnVYG4IV-gcX;2U&CzdRMKYo6M+?)>Awr_)YxwI)vrSA1ESJ$J>0MEJ1A zT)JdZkNeyHxYiu3nUAs9a_9Kmr%a;3Y0@y_dC2aupt9!W^Pmt)V6REtfjl@r?`*7D zjs33C*@}fC*|Nq_QFc3aH&qAciz1Z|1x+0`rU~rky_z0ynK=!D?(J7Dyw#Iw&`nIg z(n=33$U(yJ+r6qi4XEbE7=d?)o>e9}ZR~f7>gU@mSNv}mfpPd(T^eZr*^pM{6M3B| zNln-t?)d0%nblV}Oc21@Qs20WYS8rW9oNV+w?Z@(I%!IhFl*(vRz?6Fpa_l@ifP{2 zt9!Fc%ZoQVXa3rlzg8*G4Y*vcJ$ZmEg#{*rv(S_xx&IyL#AnGA+|>A2(iWEw=87N+ zyLZN`Q+yG+J^0rPajnzs$f!QdycRrG4;K6)PH*;qo)ZgVMC|3|EMz+(AY<|x5g6zs zdPQlZm^JHc9ZNQZ4pq{1F8&@-h-gK5EwE-z4Y z&6wSjLC>k;(A+A;9xF*1e7AI<;!3~(4g101}_oi#;}<2 zW<^NKw}zW$ZB~*G`?naENrrh4bb5;|wwPyE3q@aVlQJqMPHOg3`2=;8MmOzfDhY=c zT~R+OQnz|a=V{;YL8MRHupi4lHK1B}=OzrKiBwS+=A zB@=jR9NcE0`@Fop>+TZOP2mlW>hH$jGw)exy^AlIBp}MT)IAf=eWXj=@1mM<4bOzHq&d`a5L_#7VkJ0t-1Q3 z8=#%`VP$1@GV)NkX>u}|5*R%Ic$2(%@;v_Igc+oe#c7Qpp_$62KYgLRdE`}V*i=DOAbHAwdQ^DnnXdp~mUTm&KQw_qg-)re%fV*5cA%hd6 z<3VS)Rd<1}U6?Yi_dYybaJ~J3PZ|;777i>>%lP0fT%?#8{_9U__?IylzwV_AsuZxf0#!FM!g$&ghUdUE?&R}{CX9Rh zR0_G#EmJCZz*+{mU~C`t`1 z?CJT@`%g-}Cg9xH^8EIn_gqAv@rZf8;Q%HZAtk=hiH9bGDyk)IJ7hY zv$VM3UWM%aC1~^y$yvij**IDR*g_diy?xlhkNNOeGUBbwSWuaSW-0aBZl~uO4%Ew* zZ3oZcc29by_+DC2l{A{M8h>r7us|Y+WdY6F7_!I2p|PsPGs&+CIg;KaPBiJfd9Ixb z@G(Z-TV`n|Rk{~Mu;GQ@J6jTlGi=ESIEm~(si*W29DNT;_ipcI(E7J%TxY>e>g znS+q%(0FEGzc2zx9CQv_DwK$%;tn7Yfu*3}v$+{=a>`DDT|3u9F_LD_Gxr@aP$B-h zYsSKPwF!fMB6hIbju=VNm6VpIe##uL|6SLJfO5~DZ=ZVC)?+cqgLiz(wl7Qdv&sHrQ}1-C%#Ej&ZW@A;Wxv2cip@U9?R# z>q(N{)PQHI+PyyS^9BEU?K}Aa){4F3UKsoWsM>=V{H^ZmX&baQU5jyYO~S^?9&{`7 zkqB;|{z*cAV7ne|hJ(lh$hi04;)J>>;IxD!*Hf5!Vl~kJ6c&}mXEkROL};D*aT58( zp3%ck-@-Zo*hJ@q@o)WQjash7C8!A3KabR-@r*b2SFF~YrSkQ#6mA~70+MZ^=E$P| z*>l^&o*vyl7kDp&vygEA)|&K{Ha1mMpv=Y3Ju_1})|nwxhOFlX!1!}?ObLpsOOk=w z>7ZBJZ5gd!9OTmNKVcHwLI^-=5}V}|Vnoeo@~4ZpA3DA58sm}n#QV#R=jcm|A$YSp zX3i>v87dzHOwhEF-K#Vle82~1B60tV1r5eShHJDa>9!M&`pGvEi z{H?`Z-VNHAMtAEa4EQCy@7q5X`18%L7{v=aXBBONtLm16zl+^6U$5nplkt9VKd7Ss zSGS=li2;}jbdRRPL9)8iDj5JendW00H%U5G25!JOBddaDpNO6y_9+!c{+In5F@^Qx z3GCQ)!~gN52OzY3v6LA2<&-CO$nZy3L1)#kj-jB3Q0pRb^$uKISr`{y(B)U^dCfgi zr2P=-a~v{XUo2|p@dMp3@~v7h`{v;4%v)MPb$zi-A?i-MwhM>MGo64qS(%a!5YyYS zYZHQAQ7w46q4$Pu2~(=(8P}YQFv+FjZ~Mh6Yr-vLm)cQ4kz|n+E<}XO^6inqSI3m6 zZ}FIG@#v9*Q#r_T0NWV?Wvc&f#P|yN(0hDDtlg>JT2UvxVrXB(331 zyKJ4HX7<#;`ZakUwO&2PE95`X^Jc?I22c*{D^#T$w(|7x`kRf*oEN>)AodHjgBR?k zOf{-XRGL={ImCJ%4?x3o3|2+wMvT}5rnodJu1)$Icnj^b3olkMHJCB+w+$9!bNiW+ z(}pl_s%R8HVALuIK(D=^h4CCmUiSR_ff9Om=IuE9iq|1N>*d<(le;dR{vnOmujMYM zxU>3Wb^$uP()RA+(X;klFyA^(*<^woboCYzkayf9T?+kqdeKi2rXIg8C4q}RZ)_Qu zmGo~F7%rg26o1^Wb!Js$%F{76e6wYs7g!J^0xa~9%VQO^6qb8vIke1-s9ft_xoscaS+bPHU3kOI< z3%;l14dDNF+gE!-y!XzX&z@I3l^XB#q^NxCs;Z%(ID z(+F5{euOzAtzdovCyVsY3>~-bKlm36cG9T8Elk3s#UKej-ps;$n{Ew4C?_W|U{7#2 zquqkY4_WXb5oEzO%F*y_%{cN)%Hn!rQh-9;WDdKUom`?Xn=yp)ZQSCnP znepZ|5(6xR2(CYv?R09wy9s`qaFt>wXj#rCFN)yWk)pU0S?1sne?(KB&qg7xot(FE zB>5pFPMv*2*B##t@cpCWMBtBkNvH!{q($%v7Aul5k+37xhbL2ZZS#sth=Avy zCFX%5yUpf)D?wVH|l8^|g4wnJbB=@B0G#U7f?;Ap7UR6@a z3#pkaf*!T!+aJ)22hS~&heGx$p6W&Bgi!&5kio>>KJj#u{AQ&b#gLzAX$~5BKq@dHhYWF^11gw$Iobcz#Gq%MW zt}uZnLnj!y7dqzcMl_85b0*wc!-6}aB#`fNXF26m5YbRKr5lPGi}A945> zMemwSr*r+rm1)cJZ*gwV`f=e_C_xgUwaEJPPk}v%jo$m23-GjL6s7b053IkEZwDOG zhc^|DI=eOH^2PjZodi-?1o%0ahz zH`y@B-qS40kzN2I3lyxBmwz8bW|C%v~Z*BT4?bH>xM z>%I}#h&NL4tqxE{y}MwvTfaMopMDSO7ki=yFIxIz-7WZdGj)>3#KsP!pkr)XolzcWkMi(D5v|uEV!4H=?Gh772^||1hT09B*_{Z;z8^I$-jTm=5U)*oA+&*m^?R4mQhX*}=r*;GMe85FGF2K8O>HW#t z)u2|znk4Cj4|h`$>5kM*P|CkDD!qOl1e1C%g52JF)?uLqesrcczlMSw(55CNM@c)>^4EW=xmu4qU5u#wAz9gxoxh1JPj9RQ=p1!8xO&9g;NPg`H z)cmLn_gyk$`VJB-%?wRniR^is;qX@CPz_#~dOyTnBVHb;CowP_MIeoM4`^>^=1NMJ zYo~inp4*(KH)_Ms^CXw(%%)etmKsq(YRq2WU4RJY`7iZgS}yy0gbcY_l>${7@IJ@u z5;e0t(us5g#0iB!0Q5UTf6~m%J|JXjyCM8tu!F8L88jWGR-UOqF>?W&G$e$E4^{i5 z)|{Keqd)9cL-sH>^x`u9zVkiA{XrTjw`)vbG-=b=a_?tz?8a3-0C}J*vC~oH_E9pa z)ZUlI$W5X#6PtGV(teA2e5iPOd?326_e(qgzYMEsPX$O^19~P#H;>3c;(GiJ;jP;_ zH{%D7s>gr)*yAwZkMH^srZ>PZqA6j;=XherA|d!!=FyO91q96FWd`%fkKPRT^fn{#uz#2)of7|kW9la z4FsG`Q@fH~*xF(+ZVjJ+cnnqy!N&%n1i3ATPvW&0R^8%1G_v7J+R`pR4Bb5VrCMwL z0sPl+eJxT$7XA`dpaECKKU}^A-z;DY`Bf|O#Mu1q5YNQr4X2?TfIbBn0VUls>=4)< z{*eXeD16^JREAy~Ok+br_%Pveo4vZnxVoU!+qf%|+4Q#~I#j0^*kJ9&-GF-P9SFHy z>|HUI>3;H?X~_vZkU>qItj-%y$G3VYTr$mefjeFe5JO9sl{nW`&(H(>@fa(cS8>)Rg&@=DB7*Ju!CyV6C( zPjqLKMWvk}bOCH4QhT35*;jOtCCMHC^s_h#M*hF(-4$G*GVQtAi8k+M2bHk|awPQdi&g>Xy6ccq+>!W~TIjE^4lN1l|;k6mjbV!?`9sssKk}zgwFP zdJ}-N-=-SPm0AnTYcbFWY~T1ygmQ)Js0EE7Kg8bOtw9V{ES2&&H@@Br8&K+XKYema z6D=0$)SX_QUn*e31G-IaPz5mGWzgu)q}P~wd{$F){u_!1$*{3ayH7KCp#lv)OCPje zP-}tF5tBl0bwp=Pjc8BQbx9F99&}7nSkg4(`~n^pO7asT5X8Xj*RyeTKDg69OJ!^d zn%Mh1Y8J+g1LVy*oL|BE^xv~(+$&!8=e=S7%1-&iEbA+j2ux;!T&jKd#3vWLPRN8_ zv>C)GrnFa!8U%?zbSuzcXE%%>1=91o-Y1E2Lu=18gJd800$o>D6QP-K0O5>wN7)-B%70PTJ@ezMI}*3+(JgZ`zaU%)A+^`k*fl>T<>rhz5KVhdl1|V> zeqG3TR9(52?#B{Qg{6AeC-+6xA^dq6flA&=Xk+t!$L7u!&@^)2(GU?NSmvPl2@aR& z?|ew`IDFbr3J?~(+d~7qtU&hv&XXU&8Q{kUzYG`St2c z`OwG0jEQ(8g!?fP*$4iu<4EE9Z^idBSgMW8$Fb{Aw87lEb#JoC+eIg9Cw$r#*^@rf z)cw>OQ9}BMv)SMf{+q$y3oHfIu^-uwkuZ`N(^ zrP}&`w7qp$R_oI?e1q7e2!b>!snSSD2~r}cq)3Xibhk<>UD6;8($ZbhDAL^^-6{Fa z!u^Zqd*1iYcO3^C-NL%px?<*>bLKkd70>;)KK!^-=kq%LQUzOS^Dq8cbZvUf)A=qd z^&5N*G30SBTjC&w(lD8S^DG#FHK+0e5?|k_#tmE$FavqZR|H1OEw6|NG$XpwYOe1) zJRUfDtK;$TNk%*#;*#?6#}f+f;}h0mfUXdzi(+0O^Jj4l)0(L<{1TA9+@~wAg%*31 za@fJR{4(Ute?_$;p@V5+oy8qr7T0ZJ$Q3V$yq?d>sWs6kv$(j+*dE|OpJVd9opxME zHGk!C9rrE#jlnfVE)BUJvcFcqBwq=`@pSAtNKDD4HqojEzDwTobCqf&UaxF8i}(V| zS)qKXH^7tCsHiuyelg_{Xf{$gK~+x24NO@)va`fOTj+tEyOkXl+e|44(2NaW6xMab zs}FsQ=g{R(m{F*JDKH ztUgco&|?>E%wVSakJmX-kx>LFYI+=ak{L=*9q#!hCmQY{_57E zS1VtftTi=)QNAtQ69&v>Y@=RRV#1^xFw91>co2=S=?GX2KrOAe4jH_81eH1Of5|HZ z(CYQQq1v0i*r}j6w6O83Q<2JfAwM9!0wO$}X0pd)8#YSz8)}rH;?E9NwMIODAz1>j zTqo1}vHAPkvrf#!ygzM;0*F40Kjb(EC*5!Ov{yMRhr;pTH+8R94_@*c_i2pmd$uii zsRL^-CCV0i`a|au;SzsNrNiA6HIAP9$GAJ%j)c$EHJKNlW%CWHu~+22(Kp&629UN! z@R51^w%g$yE%6>DkvJw&d-`*veYBm)T;;#JY^r{Xf>X1eBMJM2hvVdl)N<|iWMelO z`kh=-eRq9#(}+$xUw5wmm~+^2|i5WhC% z5qjnN>bP}6VS~N4`_z$g#cs3c$nmz6Tf=_2p4(K`>Fsk-RqY={(0wBKFKOvwiP~lg zBwf2)P{`Axz&`!m8S9NXuSKeXn>8TP|+Q4y3`e8N{4oJjeIzwHm{>HcH>>Zlgg3GnbPjVO`1>9L96gKc zyWPCc6y(My!ynzL&+?ZRf@3E)d@}d=U91E^dD^IdR@fmS9}|H68xnqRugt=NUK^#9 z0yvG)*eB_^P5T}Llv6XWcy@hE`=`lv40{eGXcd_CIX$ z7wh_wBe*a(VDbwTG@Je{Dc`udM@~xaLm(@v@n}rVF97nYJu}hP!RL)fI|rSef)ne&R2iE3J-3nXNYdHMD>douiJVXqj8Hg&=`kuh>2L^OK zSB%p~U!C!t&rWq%BasXuw1jw`+N8_$r1rxr(X!)Zo5WflvCq*_$EO`8pM{fG|5y~p zXA>|z%>9{ilKqnVCb&@acOWG0Fp=*i9zC_Gf)s$|mzFpcX+jZ@rH3Em`2klbjtO_E zW6Hu8VPP zO$Pbno}O1c$ENKgedrON40rZ8Em#8W>y~8cx2(O^qHj zq)Vcu;@c8A!Q z)rU^vvz3RR0hW%n6J9OSC7g0y5i_MzchzwsaqK;V)-d535cj=#q3^i_60&bwo11Pp zu;%WFpxxzj>>vXDoTb#H>7kG5mv(d~OKHyxKUJB=nunv~R-fjj&wd@5a3rtfI4^yD z)2!nqno*YMk^Z0}>Rp&ZPHuX*e%_^D9D@p!a(47zp2aS?B76;`jKxV`A&KCEvMz6( zn>sP47T7~lKd_18Al!H(XbxU0T>T*khaRI}+adoiOOaaj9VqaVuLAM>x(*OIP*1TS zWF8qF4vJ2Bi?r3o#SQp;JU*>E=)p9DBeVcY<$E4XW;oHVH^gFIm+RS`zv(XZsdPc2 zMALvrC#6yN{*sbglEpu7rPNn@&2hVqoM^b6%ZO#aBhG_5H`!u^@fE zzMi2-d7tXL{_!^`jUbPEEHX84OBht4vm5L`b|FZKJ=(1{DZl?&R`Xe$0o6jtAtu~v zs21Ai&87Wsdvb8I_O;MLQ8B`Z2-0?3NN**%+!iNh+IpAN9**4~=m;_hZx;+R&wmv& zTJp5qLt)2MNWWLi1?|qdLCp}ib%u}(~a%rw&3wCfo2Wo^udN>75 z*r^`YT*O0jURU6@CME;Pj^uO^)nM%S6MpfJ-llpkJr7Escr-gj|H!I6jU9de+ruw^ z5gm@|?{%}`aWMb-bFSPCCwANyl)}SHZHZw2mNoU8;XI=5Kbt}^oo0wjhAmSn^$^

hQ~8@Se$6IBtkO}B>01#C`T(ErAmie63Cl7$i%GdmPs11 zom6THAYYA*O;aA}3ne;7ru%m3(d_1nKXE)Q)`g>5vg=9D5Dgpf_&O;?#VZiFHc8?iL~F&eL?EZaQ1J9u zAu-W)iRpEP$iqR$rGL0uGOqTH=-&XzkGIkdm-|D*8^O-K zj(GF&uTNr^$k;GjimTA9#8v#)IxsuWL!4eN6I*+u++S-nP*#u4YUDJX?OYvt-s?^A zyKxWE&EnttktLD9*qxNxUsOBMd=O$zOkQqHV;sfZXCU!h2-lQZ-PI`KyzTPPOY4=x z-a-dl;(5=P`WtWv*j<~+sSzDiA>)L5$UrCosfS(`PL3h3hKi?lrI2OL@u3F=5$k^# z56v89%p9T~(Vx4WJEUmV?juz4$3J+bYt9|hX-|jdle9vyfBk@t$BTO=Ot<}_AENq4 z1;*|>UEASsBm}9A%fze^P)pr@s$GPH&+=OQcOP{)?-hm?v;H#p*fsyf0;)W9DnGCq zae};Zp6&8hjN0Mm0Ws)F>s!PL@tfH?+ifJvEKi@Evc=nh)cLiIetxX#_X}LdKV)-> z%GI>g6+e5k8LB@*zz)M~+5Q_6sF)?D7iPYteW~DHmXZ@Q3F1TZ622MdFlP5nxpVMt zOzrWqSPf{X2x#B_?JwaG)|GcSzN1U9^VC-Bm>*R$ymzK-k183>HzCcS$JMYf)DBS3<4<)0K=}GdOPN)*034T3zJdJmK)zykZ*OIFW z>LJ;C=1r&T!-Ue!{rz>JbP!QVe`oQSCtV7?^wIBM{A-S{dD>I51)3|EgW;f^mvekT zIxc>+NB9JLhP7)t0aOoy1p>yrkCQ;?UCB#*(R>gIAct(r{#4Dml@(_|*n=KeWvkU` z!vx+97(%%Fyd}Yr8 zMJIC(6)w=a>y(UrBSf%YXMW?4<_SMw5={#M5s}^7f#S@d9twn~Ri%sqGpYrJ6{AK%@YV=zBUuP2 ztiJ4be!}6~E=8*1iYukF!wydVpNn|jwjA8J|D^03|5wURt%TUKGm2iMq6F8k%8QeN z1UYm3YS3Jp+}N}84F&2;Zx58QB7c&)Lq-N_P+H%a+PY(T*u42re{j8>hCPm!LW}^yQE^qFP|3o$1r3ycB>oLj#m$Q z4BonCdK_ZeP9BJIk7gsb&v-P9fg&Q+JS3;6c38WJK%4y|cdg)!ljF9xtDauXc#S`9 zqMuSSA?fgf!pKZN0?@%M6yuzPY6|MxAJoOEVDTQx(mOIUTVg^?nrF~Y5(S~RT{d!z zG@g@W=iblbuA-LLHJ5)V)4fp(=U-O)dE4^ZS#z*v&DXkFBHIN~W*{D5Z&@Y6Ahzff zC&*y4`k*E1iMi?T!?|C!2c`%@yT)ufdFv86f4#BO&8Q{QgW=TpAHx^(tI{h+K@(grHU{SN=RLN}#RhXLQ18YgSRS9=@@z8SrNk=@i%(2gA9nd!%PX@csHK|GoA)(H zv|_i(w`6G{!ppb#_{nt$Bt$CX74|hv2W&C1Nlbdj4D^UI$*L0-Fi0cNEq+zO4@^?>H`rBg!5@|uv{~3ZSdwcoDu$tog+GvG>c8% z%d{x7@@Unw+PxvoQ*PE5PMbgT#6)4R4c+|uC5PcdHZ{%yc-+v0cBDcEV?>~?)39`O|E<3^;RL2Z4HgpQ&F3j4N%qE&L`X19xkk;(HZCR z^eN>uk+_Wc44;7HSEC598pu$*3u8qI*U<{ymlx7a^L&)z(U~b`WPu0ul|YBdF}Pw} zBWllLeq~=pCZgD`b8cJoo~4l$>Z+SGw6cQT&+P-Qwx#cM$nq)tm)ddUb!t2H+inF) zIFGK$vWOSNPa&M`bmhug4ffmu!__REKZCAy4~Qg+D?DnFS^^yYQ0)ab-*k6Gp~AQ& zM1!iYj=)tdJgo9*!UY#3`;Lk;q4FzP4e5WyK~#Ft(89umxptAS-ozj zeEH|vD5Bq8&YxBVF-u5YSyaf^2DJ6S!AHF=MIq`yq5HBzS?j{@Y%pKDiq!xKm?+^% z78qNSk7F`S?AKsb$fK5 z_4W(>)Hyq`fzxw5?lQQK#MCtR4uiPPJvz71oUmZKzokjR*S@>^JbF{}%x&L%nBZUUneZV0B9m#~-8wjm zib9!LtgU_H5W2o(!bY-#H*I-}Fu89=pRw|t1^1J@o+94*T2#x79mjwBe4Z5h= z&=aV{r9)I9e}&d}r1FQPx0s{~#y5^or9W{di15V3T=L@fh@=rdHTv<;DKEqi300!; zA(NlQQYe{c=&UojdnfljI-~_b2(9a|QHW;R;R>=8<3LbqfQoT_Yk|qMX!p|XYs5#D z)Of_12tKeuu>7$*Eq~MX)aF4Ik2P0HPH)YC%0;Di z+`HA05qCm`2#DNM*u?Mxm^(G64KtWL^U`mEad#GO9C9@3?Z_}Hqj|e64^2!4EwM^3 zx|nbvCu)k@iF_-*89=e;`<&fa!JRw1ox>K3n7h$Dt^zLK(7b7rG+^^W?eju^^)duH z6;2(m5doQj7+JT=F3$LjH2#7I6MD9c1Zl=^g%zAAu+nSz|lDUk-Iuw66>&z4a6*u zwz>^DRi{b9TG2Hr0qMdWS?UA#T7(sv z+N_%EKV_nB(qJ)eTtBOS8A*8lPN5Z|598LKS=H2&A}V_5L~H%~x~78;71%id ze3$n2ge-FfbW$3$-a0l7HoeBWboD#zukwfFGl@kGP!!7-%CC(&Lhl8`#v5A=qb6r;q747b}KlLb?oHy1VOC{Ri1xcSb6}SVh!(N^{>#mqT-01 zr*y-A_%EpZdG6!AknR6X&!AS!bOj%vzriD(NK)u<3VW zLVg#VczL}gh`Z@`Q9atbIBv-AfMF>)(~vQ8W9U}YdK9g$#xrw02Z#R%i6F-b|0;!xhMG>gRrqQ#T?*{%B=m zM_lcXItEUhes^UH^mL+m%VmEh!&+BE_vZq(gQS)A%on=MaMo*2Xp*R#ir|Uf4f|gjP08>BfNB&9W->ov2YIqaWR97eiYl*qV!1bvv?(zJ7nFm z7ldg-hn!Rpu`1|Nn)sKdHl878gr|o}N>B|Z9Z}JfC#|T#|K#>J1Ny(@fW0z%XP0Zz z_g0Gxkd~YWdG|p4%lhFsHgQ*f=*0dL#oDeE;r)_3L>2R9pkzSSkaU#wS9k0RRFtPr}AV>>bjuOB2Z4Z%w zwTN0!mxiu*+^&OfYHyg3exJ-rHL8M|iU0dULd{2ee^_EESN&H$gP|-U#~)f`KZ_MT zkj9p4!Jd@iz5x#BDNX0ErwLDMutaIR&3t4ij~B@i;T7ZLKe|VDcYKRpD$94SF)$CU zZ1bU;Z|Tj&+*)WnOQi+NXDr(($U)E8sD97WA{W%nfK9P?98HQphXDAyq`a6_2FYnG zh^{C!t^7%S3H_h*^xys{+uXl~w98}>`(N_2REy$K|!&KOD)^Rm7Xi-O8m-i^~K9p1~zEXCud>Y_?BReMK z(^9IB$WTq1bjRPcaP!OM=ditE{_oqXtPf81ygaQM`}&kRRI4Y>`9dH0Wx(3`Dxl)8 z#U$W`RAW=&g`GU+d?ov_NnqRy52x}2bdkCF3Y`4naAh|UgSHjWp0}~9lSO(eQ>O5n9gQNa5=A~ecTvX4$*pb2#rs+5Nyb1| z`*pNw<9L#^U>=LF>^}9c8;btYbwL~T%sTV97yX0MOhAn;`Xd5`bwMT*F~tx5tKo^; ze3x{U4bGxP(W^ptKV`3^C^a1qFXv>yAVYC7uX|&s2jX6hOfNOw`Tmzx9x>gwHxs@5 zTn1Faf6}Cg&l@)gK?mk<7k@(*_#SjEHvZeSxUi_UqBNeu3cnwgm_?@!>)w3XdrcuE36AicEL3Q8e>*$hwA1ELRSSli7#s>E54W^u z4p?!`SsSW;Dh;2h3B7h|7&djtBwG7$0wVb0)*cO7#^9@Gl!S z#6Uw1>Lr5wT@UAy=@N-L2thfy+yzmZ(vXe3lIgtkfEM0UrKVADS7a5=*hBkZK)b`< zQ4Ru>dCo@qKZo7>z6IaEi5I9#LVn|YwF=vL-na@Ziif$F7GgAyXhKyE>d4_VOYLs^ zqw*1N=$qar9~b}}9bAi-|B@vucJ4Sr1L2_K+`oK4fvOqln_7mL^s;0r=GDgA)7|4a zH5*|rp0qjTq#vJ8gIK<=i@$>S!KimKK5#dDyp{Aw&r5pWmAIv!mGUOW=(vSFi z)YWtSSsl#{w=cq=#?l9Z{e5oK@@GL?Ey3J7}m9EUI*lGI6iM9&-PEz`2b1#;$^-3 zsvkcS0XnWC+6TdF_gjxri=^Zl7-sQcq38w=)PGXn{>CQ2B%hPL zCBL@{VtR{w6fP&_VQ0VPcZNSVi54e_++>u77FdLxh;{L<4Gu!OCsj=hnGLvp{^b|! zLI7buqE@uF_sNIW4oe`&Xm2&TQdt3mbaq9%{=FUD*LRno*JS4c3-aMPK*o!I!Y*8h zOqzezF6C|W>z^No<@?WY;&UAD-uE{nqzXK4PI}?EU&D%VXumUm1dGyfdw7X5c(Wf9 znM=5A8plTv*g35yv$DELggmU`BRRT0e(T{oy4PqPd%j+oE~esV{SIw_{{BY7SWvk= z3VDyB+;B`7l5yh>dKef)lT-hO1HrXGqiYF|^0rKDv;gw+4>)6ZZgS}L71t_e)xj`~ z@LOO4^tbp;32`l4fu4|zfj4+4**n?G+!JQy@YDUJTVODB2$2(Uij!h!SPuLc&d1u> zD~yD8$Q8Ixpbp+z)6KBhBmKWtADI6S7*I)GjCGI@=JkcC@ z>sh*-?a80Hz?c|cP^UWr37+?I3=NIGBxe1pHl_Ay@R*f-NF%NP`usp>CA)1p?zn|z zy*7r0+TWk{HygF;FQg!0d5!T%Sa@T-9%s5+Mxfyw&ckvdh?x0=9C*$P**W-cIGJ3qH!|CxJ8Y)rL_ZtJY8lrVDH5;Zb`tl|5n9HdXveBDw zorf6t*+fQ0#yJ$m#f#;y|2?Q8*?}vjwn9JP7wP*Gf z?LL_+DJi+%QNBOW&<`g*|9w3_t>?SMrz9&Ydl={BP8eAi|GJKGbm6loDB?iB)UpiuEF~K@ESIOaZpQxV z;oP?ATV81vXEh>cs2oA&XJ7xc^xUzD7Mic6hDNQYDhdjHvv$eJ$ty0fjZNo4SLHuf zb0snYhzP^UJ6t?IS@-?wq903)xa7le`r$gfrWZU z%%;PorSz|#L%z>6D!~%zS9jq9%e!DT)p?l~=Rv@Q&dw@FMYI@7iO*|&ekcJCHm z$+=W9LnEWdum?VQd*hLic+2qr!C=#Ae@A&~f1`=Q>~N(xF*_T3y2-!k{ke-nVYdq} z8c#_g!2W-I{nIl0nCz;XQ_aS76N|-fDZN5v)VsJiio1kYm6er&Ml&-T$k)ZWfJvm$ z@BD}98G>H^*VjLdr5D!_C9#0mSikmmsS!}!yM&1;A}fmzH^L<+_q#~ph)YaNY|VrX z4(;DppzaiA9e-L?6*+4J=E=!PHaWtX{rf5r4a=3Q=X?)^2FQ}5%$J!F_QH8sQeop@Hu*Z46>xqg8mGLI0WaJnAna`NX6vOwNf zSG$BYft}gtdu5`z)R&c(nJFSDh-&mB^DbYjB1shUFPaPZ7@hs?+qJ{pmHM$thdi4# zH9hv5s2Y5khwtHM%Ek76Em+#Z#O7JAD8s*`^%{V@;{@EW0W90(`qD?0^t?R0i`b-k z9nXx7??X*S(48mi2TM6N4l9%Oo_V$#Q%`m`+SxiZW?Ms5$%~clB65D~y^8Abxw*I> zC@Db&vF-D{dJy#k%6eSj_TC%zn`RR=Xm3Ha0xr6H_Z~x)Bsw}e^Xu=I?y8nuzk1_7 z9H=dwspK>s@$%%)YmzHk)aQxL_7y^HtPSgaQ1Ksdu{SVmIv3fy&Ig}cd*te$tR!WHy) zgfe+36(n%~iB5APhVCJ!{cB*M47J8W`QGWiAS*Q7vIRaG6xsjV$iT`O&G z4Q1s0(U@~8tFAsi%U5JPiaec_eT)f4uLh+8+^yQnIJ*C^BolKt^qC?aMFH3%XsbO)54TpQu$k?41_djBjwqoSxRG+gLw(Bk{qF zQdGicD2iHV<*A{Z@7K=H3Cg`Mc7&dPRSrpzd!oNmY1rpC5Je;*a_u(jYK4o-{^(rP zFrRr2=B7PAxn03ej-OVlb7enE*DOZ+xi3nUN&krCXwx7EF`5_E&Q7lSOT8Ji8yg#( znH+&DJ=v^LRMkXTOib+0pSd{S&$UC_BHZbq;2Gf;Z_xWy;?0|+vF%`*r>qOSo$0cvn4c#> zrs^6IkskBP`ec26+OEh;SAmT?!OH`A=O`T4(Ew9CD>hbrJ6wdNn5|L^AMiZy_2Y+5 z2iHf-tm;cG=FH2|M7Y1gYAmss^Q#rW!NI9qFdiv+5Aui;U_LKlV>f=4&oG^BC0HIU zzj05w;GJ?-BWO#0sy;bjj4~8?L(A;foq3y5C?T$DBG1z=&D?=Q% z_V!bV7G;C?jg4`aQ_Y%pXeb1MXw_0!u&l;4%3}E(rfS}wGc`4>f&KCk~;sc^aj&&QsuYXzjYtm-*85d7`LLmO`R_cM76W3+v+phhg6rsR&lv zqkR@0%lUu>hQ|rfQ;UnA$ocFwPLBQz6dK;RuUh8Dru$Q4XR%jrq=W@E0b8>UaN16a zfXymtnbnd9txPQC?b{fqM_V|^J&~c(toIOq`m`Z{N;p@m$?vX&Oh~sJ1A%g}v2O&M z?rWF>@G^oTA{a$)2NN7|xSfxuHGFZ(A!lmdi$~5X05Kj{{|ux zn*1cpbfy_k`_$rhd#w*K+uK)W37H|?Y#||s$*HMN#KpZ(0kFxRn45o7*axSatk;=< zE(kY?5Kg|``ZAKhEhTg6SE_DDEV>|I21_`tr;w2hV}dUD;lqd9ckVP0KBj3^U{dE5 z`JPGO1}e7r6AcDKum~62a7Z|%P+s}@`6E?M!@pruDH|Ak{duKdGL+$E!3Smzj^5>C zlkqB52?3WA$C&lU-h}j%!$n4HyegwvE zQNnh3FTLNHC>ARJR5n$*(9S*~dbfo(t{HX^mh+95g$ALl9x#z95Bzk_PlCn&G3FvE z`3YDwVcXKf=kMnA<$b{2t6gv~-Eic}>`d_lBx}ZzaGx zNAo!-&>q3UB|uHBuHNtL>?B-qr7Jf(Po=#GOBtpTwLVqJE-fvsJQ*7sOT=q^7o`FI zw*C=VP8KkCPy^xTIB2Q@cmT2Sv^6&q{L{frq@6Qwq+bX;K$ z(`k4N1NR&hsbbavsn;|#oQJKfROMLuqZ@yg{o+MWQm5)T9nA9=FEVBrvn}@LP_%~7 zpF@G~cwY3`xA}U8VvZ=dGb|5(GF}B@El+H+uFHVl#{;9IF==UO9glbVc_ujwdY*0e zKe&+j6jo-6Oq?J1?)uKo=O(sc_f?;jS?Z$>ngR+TnDB)ML_^WY#A1S(%rbRxad8BV zGn85;d;430ThFCo>oXJS=<0gHW=2N^1qH3owBTof(88J)i~#B#Ol-Q zk&{-lawN;l%1V}_jzfeFW^z)J#@^~EC)2s-3cM~S4^fnqlqfYdHDyhJ4`TNAJgE9{ z{kpok048;QRN1&bOu;t8FCQuxUvKp$jb_kr4}Hwy(fF)0;qmiiNdgq9@b=;a29>tt zmRha(Y-{b{;7gNe*a(Tw;vXq*eB4Zcy^h#4I{B~HVgGvqg3ws(N%#8rktJM|O^1|} z6!+#$1LoM@!WWu|W;uutkD^Sf^qJLqzsg%p4O8=VF8z+N857deqx<0M`aDD79wrGV zCafSfA^}#`YyK1h7c_>5KKc7M#qip^eWjOh*Lrn?`!)>?`nxd_$w)(2dM!#V$5+X; ztt+5BS-46@l>c^+wdbq+Rd6RFVDdW)4HW^%$=P+p^G8-N%70DT^JG9ht8%m}DJwH* zptUCu5fjsb-4HI|!VgvwmSZ-)Sa&M8WZdi5lO9lj+trp2j)&%AFzWeWzcgEd?htVp zTt`hnOe?=CyJRulh*e-bdQ-bKL}>7vc4vv1mQ*-%?Ys@qnBtdy0@_RTvXQ}BFrdDT z-EQL`_TJVe67ov5%#L_Spx`up7Vf3Lv-lZf+O@gM;z#JEGI%T&5^7 zfAR)jk_Si}1H?fhnAJ;ISPif&c2Zb&szku+q5P;?M>$Q#d}?Y0F)mz45PXlmzO_{k zI^^$_3k@_OyqoV#^pw77xb|aXjJ#6Xc&sAbhl+xNVx}XG;`m@k81C`ABhDX0|A?Nt ztc~#@;2QxVU=#CnJIJM?x$poV)qx4#N$K>V3YByRt>|znw{u=(kguE|X^(tx~fcnJbWITcKB9Waf(r`RV^}aM_T$6K%gbvm zv9YoBhKue2pz&nYZv6u52f-F#<2=FfQ^S6uaN5QLr^I49A(&MV$uhaPXt3RM9LWNR z-eayKj?q7twl#}ONIatJVdBGl8*IHg4%p*CF#Su8f^;vx-3Gkrj?hz9OMMLx8nV0{ zU<0lTuO= zq+@vODSXp5U?vtFb+2G!i^6aX6`S~j3!r9TxT5a-=MEv0S|)-2c*ML_SDBTeHb)mo zxSGKpYmen4K|Ty(v<;Ah{{o9!SwEBP9N0J>`sWF5tRCb7&OS^1In#hAYZ1~5A~{k4 zXHuXYeo5>tq{c1AjASSmG92u#c*FcrLZk+9ttc4W=U)WT?)a5z(P`aG81eBZN|w@B zc&0bjqUlBRFkk%^UZtPUF|cPJueCB6)Gi9?p+-D$2oPOyeG z16HkLW8|=lWE)!nkQLT(d(2xELI(-Ug|71ua?Oub^1}AXc@UefMS15A^1o1b%dv?i zWx%GCn4&$6WEXn-_U>2396yk|uLt+7;i%Q00O*z8UtXo$`W}QS3S8A}olD<)d*$sO zz6ML&nW4x8%Bh%6hr8|o!Jq&*!IX9zY*qW?!OdeR{U0DtdMx z>=&KZ|7?ADc&KX^;ibm$G>TLH?q-Zy_!9L{>pVh0K#ZQRnllFzEtn}8Cs74!+wCcgp!+^7>Qm0fMboZx5f!nnBT7*0JU!VHKrxYP zmsl++WHJA>V6*3MiTVfzQa?taT#k3b1{X*;43Zql*|%fFvlNz85?Q-^s00KA^7mvy z@R8SKP_KH3nwVS#Gi?cvu?UIp4mcd~wmP2P-T`5Y4B2J4KZefsFn z7K*EpZmrV?Z=yjf6SLDN5ZtUjUt>RD1UvWN$~&7@U*EI@wU)C<5extP`4b$i9ARis zvps-^u1Hdqyw~oX>LKTtva#OjYoUG}SLh(`3|ud|?@f*uG#71_2he0>WHPomW_g42 zSgsQje}sub{JO#B-kzd!nk~dsSlxAo56hQ0X-RBZjhUdIvy5?L0&1mD<`` z_-g?D^2*BO#QK!)TaRqvclvR2TpsiCwU-~Y$^kcfMxlWlN*m(>2q{a01D0d{6n2lw zpVNfL-UjX*ZwH=5_<%-q)Yvv+?!mpF!BmfJHIkUnRrj&6iv|S;=Q->d{{xWh-osK{ z=uSnOS4)L;bOxO~ns8BwyFfk5QLB`qedT4|3Na938}ex`zCSJJgm7$pA~Bg=tEQtv zW{D=~UKjo>AsC;AS0Xv;K-+_DdQ?qY+Y?49whJN{$eM{d+25G<^7HHaWRkHhl5^!p`DJx-;gm}_I0#dZ z8eJ2S&rp!3S;umsl@u3$MsCy!Ci)738zVW5uBfZ4qfYlqTzz2wZBlO{ZU(rQS8ZY; z!*}l7nas$kLPy!!+I|58r#y*pb*qUQbh!3)GlHwb{` z=qQMe8LvK(d{YVtB;iTeeQ^m1A+TgmV@WYiOfbJUHBHaOx%6H3_wo4-!n7Aqh~a^G zcpDi@_khdkfc@qJW1snXJ-HmU(Uf3UWJY1tprH@~Dmo^nuntsUjblyp%peS&#cvm>2(P;FJ%sRmOPX}9v$TfL>f;<)Vos>5(uXB`Z6P)A^-)#=EO`= z_B~H(a0UpKhpJGLAR;78Ay{2r_JM7WLIK`Pl1kCth8W`(J^f{CYioGU^79;&t!R(l ze1JYX&rz39#GG%#A37faB~8xnNXKWt{j84DXecEA{GC=b!@kTw5ch8JBjW{FDLNP% z6hsZP9mXL4l}_tTqYttD7MJP7yEHilG|DXH!XWrJZ?ER=R2l7~{SCw~aG?SqpnoF)8#L`jvLpnJJB!-v zYRbxy1$v#2K;ijDsohozaGrFUFf^a3ggxl`zdRUsl8AGHPRMD;R7$ zBZBF}z${-z(JN+0gLfRt4ie$q1e*P$tE($20Cw~I%3Q=kSF$$>7vkNQxf<_TKSTSM z3?DIg#8Q`2=g&#Btr9>XTn8<1G)f4}0!m7_^1EchnS~ntDVS)bqXPgTYC8aj7(US(;)+`B!ootO1WRm*)G2WI8T-8EfI2y@V-i#&pFi8%*<_|@Znohp%XNX z5#5U5wn$-3kz)wm-`}UBxQ`Nt5awQ4#N!Z>hb|{7KqwcSusaRc*9#k#$1^3pDth~z z-{%}I6BE`z$N(%7&Yqi$etv%Ff+Hg%1w}Z-tRjr6z<2cRK(`}_ z+7IVLCANIDC|Oy%CQg`C%aTTF&Qk&!4g*KRc)ANK1GciZ6&vo$?+`UEWVm0w(N{`C zOd@;}j9RNmtWH7)7`3Hwcmx)u3y51y%FSg?&NaS-F}12rl>OQc#-kCu|PcN|~T%1>pgedKEFtUJ8ZkSnE$WGm0RFFb8#>bBzBR>1w!gjz zZ35KFOydlE`er%*vk#T;_wOPR%$gsrGpVTz2}Jp{F05K1QExf_g*Q` z9>Wm4fgl^9q&K#=dm_Kk0ln;aw9#Zf-RLXT78o0QU*Pm8UoKPWc}qQvI+*Iayf&-t zA0qJwMdjo|j*hlEHi4@5j*E-yb25A!w?%g8n74*8tqo;X5#dN`93~rv_Fx zLqZ$S>2ql$^vqN#ePO@5Ob;|$Z{)zwFs3+myp{`7+YhZpcFfjCz_jY$b zTdxd79KERkv?~d*ZUM~=uuIwyY|}WUQnx;pqdq;I>=V%e!}b`Ww2F#~iHV7T?gpw> zd1&DUIlrr5rAkUmUxs)pXNdx+y>gxI%hm~2f@1Gzxy{p0a}ae^Xf@*EA%GW%8HCix zZ+i{+CmO+eU=Vd~cKrB(H8wUTI{nao`x1DE_4W1KeN|mpy0~~zl`fy?7)PsJC=3dR zl1fs1s8HfkQX(&2T&t+8jDL~dem@vS{+q0nl+Tz=^3wGD{3TeaiDnRyZEP>d0*hS- zt}TJ}4lV61Dk?Pi7@BIC5W3=DBM%@-YuhgOr;0qhKu%6h=zHsIrvc;l{rmTt!9ikx zvdoA1A%JiofS7Rx>r?*=5qtDK`!TkTdT7~Z(mNNf4U5NalG6)RT ze`g3oJWW5G6^8sPi%5qviSh3;0pwSnlh<8-^sieY-xD6X;k*6!Jw)s-(LzT3zphPO zX{0jeKi4eP|9ZCnrw=k8`n=NK3aEzHz#ed`UT*~nnQb9p5wJ?VB`Xrg5dQ$P%03G zi#>aW8_BN!0pNOlM+eiBHr#91J^)g_tCWiZzu(&B^m1CBelFwp$?+jHCJ=f3`nB{8 zck>!>7P{^|SY4|+L^9;XT-EJwnhF8kl|ReqwLLj1 zZLl*vf>C*%O?fu?H2X_A)k(k?pw~>OgW=Z(hT^3%He`O#D1p-b*>$Y6guFZrC8Z#$ z8F)U6GP;l-fMn&eRbHZ+f5!0cH2wjDf)yJlMdl3*D;`8dm39D{{y;+tb1Qqso+9FWi$UQQUiBa7LT2q}-{C9v)FAMt6K9d_S_ z+*iE`*sK9)sf{s*wOPx+kMkTzVl$fG=_VwKg&}vg-6#R@7dX1kN(UQg5qA;tDgG}H zk(g);Nm%M{dj_P5?;r6efB*IXn_qFbS`Lu9K$<(V(s}qR zInX@weOV-s;PL?KWw@eJorBZ`CMtw+W*y@WvJ+>`d99amd3kvOv>En=gohJVZo~?> z+y}d;sDRZT&CT|tb?a+pyMuh|pRbcw>MAn8T%!x7NJYG6mWCV+h~15pX!c#HLojAW zdbV2SSNyReqgQB1GGlXlp<5+vOwOQC`3m7(I@AnA)r51xpD$x$6Qa6OW#Xi!4HR&# z!9AJm1ML#Re*zSkfxGt3&TzCnbRUP57)l)GVip!LO)^V=M(&4zY3P0FWqvjLuT^su z!9ap);G+Ri=9*250&JT1T>?-KQ3Do4xM(QHKeJ(8zP`Pm5M&9foEpX-b~8uGI1)O+ zU!oqEC=~fp2#`b9M{#j+6bkbuyLbyD{0jhy-qu7Y27tPZ@4z^i0j=Oe@&Fs$rYK|# zV99yFR=kcw!QWGUx_nnx2l%=TD3?lv6cDhT2nUO#KrB~=2#|~>Y|Mn<#>g^V%i{wp z$K65QG{qcp$UV6S1mMETYe>_e?W=NF-K$dW`J`^l3w%j&-ECs=d(0)~@se7Q1Ih$Lul)6$ZR`rmQ72YeEwTpEzFG01vN z`92sEL>TQP34EzYb_u&1+B!Q>jZ+Vx2?@uc(kcy zNXb44-KV1*z==}<`o6F?94~xPLQ?XZ{t$po;;c2W9$Csl;lV2jgL|-Z8vsSqj+v%@q_T` zD(cU6x6BX8RPIidSZ=H_&nnnoR zsHwpXWTOHA8An7nUjMD;psvR8XEUy{va+_f)k6U{B{j8;HqMb6^&PT1KollFsI1!1 z{qZLPhPwI1KhEBj|9D$=wy_5=>{H2LK$ybXg3zcMfaE;>s%<~bZ?YK;v7=D}eZ2`4 z4sG$mo-;swZbE553oMx=m(!CMNJ`SOuSZp61|mec#qn?2PmobkskD~|cX$gKLMXt4 zP#=1GKJ5k{tVqB2D_(dOrpcWkb zT|$mep>u$p_{;|6hCW9oF;z|BpUx+40Wav{X_WO2a5L zjYxx{rL?r9QnC_CXlY54G-ykS&|Zp4N>f{m>$jvqfhWj&(T$MjM}LQ+EF8@fU9Mx^g!U(x{}*mE`u@r zQ(~CAc=Ib`j$fWS;~PpI@kWHkW1TkbA2wVa74YWcoQ-Sd0wN2UL#kP+>c zWokI4ZvTqT8&8;7|F^WCnf?Af!;FoOBT(F&XfN9o;G2H}+5(h;$t6FLW$2_%#u-J) z8QXZp&?8X{0};T?W?(eS=Ou>s;9$4i=w~D(CFyjuSHI7gWf$D-b?U$3J`zJibAvivd2X#^JuNNQabVa!PU80wEF_(mbwBBKj9ZBnCId+NxmBAoWm*(l zd%!5p%Mlg&6Z5KWazS7DcR}CPh@HtKnr}32di?nDCP~S!tu2E4ak7};{Par`Qa!b4 zo+o7m)p2hOgRGmCVK?BlDl;-6yaz%O97gj@yS`mU79z1J3{bQrxi82qQ!ZF+sylSa z7`hM3B^4`a@dEq)@V)=bE%tG4(NCjDSZF~2_c}Oe1|BpJYTL4NXS24aa3g#o^p43mwsrRm|;0j|GOQ&kR7G46} zZp|9gfiWeE!S=(>{omvO`L3C@a4py@cDJ6I z55tK+?TLrhWBD^i+1Yg-yj6KFepWfN2dCn zMpVGzXM82W(jE(?#};8r$|9^Df(Ss!xN;sOjh6!g0B&h$AiDH=X`!%X=$l{sAl&4KVGXkWBuBhqTuh__Ge!F z%`iZ0CKv3(_XQn;ktE=RY`CGl-4z{HK#QXs_el}c0KhM`wwfJ4^SyibG-~?WN<9#DPeD&fveo3bmR&}@|x6@)aVdcw#j1V&kJnKF1rJFeLdLhzJE|M=)usXxPf1gGOL>m}X zN6&BI$xFbvn>TOP)7YnU(fH$8)u(zQQ}k}Lgnl>kb+fd5TRZv0x*Hp(K&1l!T1`^S zL=l0BVIaZCff)gklu*%!2KQTA_KUV!+pD16Ch8Gtw~wR4rGa`%+B+dZfW&A4>+VnXyETTf(wWvEuv z|9}IW6+d3PaibZs2>B5=!MZ%5pY4KsJ?Hs+h*2!$Vl=9&Bl)JLagPWA#?Q z^HH>{xzcLyS-o?SaN~+~xBbg#i9%m+3Xh32DWA>?@AFt|y6tsQPOxy510g&LKh|8M z9$)Ve5I8UG;e&pH>>u~1PI-eO7S|;(h?dysy8gv6pAb2J5DLwRPBRCpYKMhw*z(Xk z&37@Xvuo+;xdUu&HQ!@y$OjHT(~_k2Y==$fKM*zD_4mI88Tfwliwd1_b_>h#0m*f? z`tAJ9VVh#*G_j=>3&+qhfBDwu^zTFRyT?tDaFw&5dZgd#!Z}Duhu2<810eItVy(ml zXH-#BJ_PN`?-=xxQ&4z^Vukru!m`JB1i8D4VIWWud7)tgC9}Hr%@glY?P!*{od#MF zYZVjwz_PpIggG=>y4!I1zs@zjox!}B6dgSi(9LN;5v1~GqQ?An#j4WUzB4^T52?ys z{GTe9)DfhZd4tAVKre4mnCMq@8Q0MV&Shv(du=%;&r=KaGtfT98GTH9u`a-peTCtE zn{Rq}h(;{{p_B!>$0kJ@idxibGnpjwdabTVOJ%Q*Cm_Wtm~Xka_Y*P*F3v?h;w`FO zFI}T8#^sQ0v%a~_#tUM&iUpKy?zeo;;)0KNznd-7jF){a^#0J~%5ugcpUR+)ystm{ zV{lLgdd?{fVb`M%3K89*F?|Ps5fymJv5xIl$gjv5*vWJD)xYyoiRm!*w@vI=RmD)( zjiX;bcM4|8qfD_#zjtCLTL$|o=5X9nME(Ilc&uUK^k8u9}N58k+adoJo6Q|n+j zjNao!PKH6?NfRvdzbD*W>h*U|%3h=R&$R??&N61|u9ZTD>w!CaA&7zo1G9oI!8jZ3 z=q(9{pNi&M!0}j_`_h-3em8qadtL0Qf6WhA;7~t9pi`w>eT@3{DeIvS@CGS$RX-&) z^>{zaoH_STrk|Gs6Xfa&y{|HG@J*AA3(`oiZi`SPB6t7miSdKPb5%S<(@i}E&nSdf z=}KKvc%@U;Lo7MjXkZ{F{% zkaEy&f~JhLKsB?43CEUN+Fj^_#-KCgFq)Vt>#;T~S7k!!z%qL_=~Km^LC*hu_JMu3 zBh(UJ1LNpv!Uz4J2!kT_VrWwC0|8y@@|B(Zc*1O8`4~Mgpc<$B+@%#tODV)uLMVvH z%&IRRt4a-JXl>w3YU-NW+Bi`sYj`dO-r$Kubk*+b3yZw~*}aF+Yac;yZHd%CZU3IK z!TfPlvphy9_uqrV2Xz*XThi!5BZ=1&^&W)y>kD>v+!Lx%M-xy|ASMZbF4r*a>geD` z39C^}B-+uP&nd-U^wpTy-C*i%^g(P81=KbmK-m+$D@4gvD1a2qjb83UcEB_LEnFor8n%btDUrn75bm zl|S``3}%P8_?C$UbLVK8WLi=4GX(cg@+cgrdfB~w>0y(d9!R)8v>_l0bbQw&%)5toyort(=;4&pkDxT zmh>eG>7roaF+ssfZKshA&X$XT7J}On`_6RyZQfV@8`eA#4wOOaYz7dcAsS1l9(q&y zTYs-o1gi*aG7VAALSO#Lmka$eAjp}VbuwBpOSw3FguDZ2V7*Mi z4s}+>@?c+-^jWU7f;fBy4qra{^r#&DtguBTj1gBo^-`XsyXJvh567<65anL8W(`Th z%ft3B{0+{2ewCT^GHLFeqqWUp(7uCI#)9HY&%&0mcVuzR*cON1m4H0}kl^$p0mDo3x7=PmrMFY){uv6TUr494}L z7cbb{h)tuo@>1o)U?Np`Llz74FS4h+ym1;huD;A`Ys+oT&dQkog2Rsf-BF=c_cQY! zWo+cH`no)&Fxng%i}spSekl0eungL)EwolZsrC*Mgg%&6!mvxlPOdJvN)la=(;-N- zY8e;evo2hp!?p)636bS&`Skv1=f5yHPkP{1h9W8Q4!8ZU9(TwtPb0d6Bd}Ip*54=iBvlTpL|=uWLC*USS6yM(L@zyXPG7Qkz+GO+v?f8T!NzkM*&&ZMyi z#}%8q|Nd!El5Y^rs3p3Hn$_BZYhglaYHqx4ZbMVk6Yz2p?@*D2w?WD)&g7$wd|7>{Ej{U-9^iK_~-M&j;a5Lzxw|Ixk>dB{rP{KW&L{@ z|Nn*jzmNd`pLsR^VY$5|L25w=vH&usCr6+1)_Az?idCJ)tX{2E>=961p=x(h^_TbY zmK^~9_sX1twC2c|TX*EWny%Sn^?euEO6HzTUb!(;MYZ_6`|*&k19OTWY_nNC8IoFT zPE3N6dLamFjpW_{bq>kjJ;Y;z+aSY#&;NStO0LJcHfvUkpWoFLgT&cDk6uem+`f9v zns|_AykFaZBRFXNwG51iHt+Z zF{uhkPY;9SW*8ywu8)twF+Zfac&BlD>)yIGz^tx#dM+Vw2JNCQa1gXFRdQBf0mncK z8J35$psK_zdmyNV;vPWPKRVW8=a+0#dMi_>$08}|l}R;cfUxmQ$UQuppP|Gv5-kqC zdUYCr*fOxB-@ham#*ClwMNt6X66-R4t)h-ly{C{8owK%ne{a+NL9?o{>YAD`KdRC{NS9ug(ixmnI$1drV0oMJOQYX(lY?g`oT`ytx)Kip5HDA9uH!KT$nYJ zQxKy`sE#k1jOA@@XZoagO{0-8iQt4!>f;wDR8K^jv&(z!0(ir_wF>MX*ppeYrcqH* zaO=-TyLkoGB8kN{8iCut1Mi!Z0wvXKe84*(EhMOKc=f9;@rB|_l-@iex7c&b?*A;f zBV{zlTYkpQ^kuJqAC0bnB=s$NA!kty)w*-lp$=?Ci!X08kqsrzCjt&v5fX zFJASNEMQHOYDJ!cTlwkw4@{N z6TAj2T`kdI{^G@p?>>0oiU$x1&=cbDrNlf|&|}1x>7bO(3zn892^3u+3J2&WcLhs1 zB9|H-OT*SEOFVNBnk#9vcr&q5iI4_IzXZLJ+ADiuH2xvpkx*YO7cp55HOO_;)En^% zQSAT)o-Z>UvWdj}rwYcM505!!-)}aQbI{)(EF_C|^d5{GD9AwIHe&@C)@6uca#xOf zVLu)!FhPD4cqkT0tpr3&7j70VjU+B|v*HF`E#_gLRQhoBHqF^XCFyqZ%%FO{itR%t za=cgL>u@tKCNmY7AIm4#CSZ4;MKsNyap+9(^BwySL zd^fJ(#Y6Fd^zQIB55cZ+d!Q(`*SgTOrCdTwL`di#tm?hEQVSq~boG$I74?swv2oD5 zc;Z~nL~3H9it6*oh=|=0M>_bfhS(nL-0KDOD$(At~9FX^ck zZL{q;!Mwqcgz)WR4}7z8a&!RFU4YQp^yS8R3?dM-t9SILkSMpXMZ!lYtX+UPfGa;} ze1$jN(5z_bmZN#$!$2yYz;FT%FfYkP*rv1~Q%51^3s34#xrT`XJ#+JgAkt!=W4-7A zrJoU;Uqs%6&2m;6cc%5tjdCrd>ttUp`e0<6&sP$pg5y2|O zdc)xTolS7Q>??E}8@2#&wH7$n^VHP+@IMgzMGo9kY1mfVKpFkReMH7@{mIu(AYPVi zfQn`h7ySCdyk7od)cPi0l2}N7@D@+$10<&Z@Tks1_s?}Cd^Sj4 z7bs?S%kb7cX3xc8O6NufjVLi7euKTMvY*hw@|u!iXpDq`*6+CaZ(so|_#edonx`RD z(hw5Rl5i?aLAfs!?dFrPc>>teDez-lFfJkQGL`e6A>$YBd zY2KK2Iy(nP6M(3FXxtic7qC#Bg8o~B-tHMWO*MP7yr&|Q<9+6`P$Kk`iB+ZUak?{b zs1k$Uk&%)9L%t6mnw2?8VYJDV^~hJz#RYK!r`|A-X_A2Xmg@l^bu*k-?>P6SGb1c4 zj1;!DU%%>uz*)@3#tyLE2d%gq9$q%Mcre?#Lr#K%5wX}2+>9Rd@FHJRHy|e84-DkB z?Ww&dX4UEnmB>GQ8b={h(K#}LMFiEV8~gz@7vPwFd}NP$^>Hl&rC`xYt&|>5*n?Q1 z_w;NgSMJnZFVA;B4P1sD@eN+$eoWe6VMq$9EUu*4HA;#D%;1O}M(0cz4j7rT0E5~W z7UHAB2xi$OCCZ#}(E36`A;aW>k4`E#`TTr0nz+H9cet^wS{YALPmFZxnw#?z?~Jw8 zS!|Ak!I5l%kcP(q{4uQ4Zd)y3qSO;3>-w>7l+1Bn2tb#Q1run-MWWqSZrHG)304KmbiaTJ;?l8* zXMTjqT>YP4098qUEBa7A;(oBxh@5%4u<4hh=Kjb7+!jh=PCiN3*&Vm?)J!6oyzkhq z5wgW`>*mefd-hC4?@iXxfNyrNlf`Ne2K1*QQv`gkQN6QP@)o>jGk^%VKvi2jz_0dv z{epE;Zdh^6sF5yX)(nWM4=|)rVuLV57Vq3Nv%ZLcR_0Lh*<)km0M=jdf7{ z#il7(sg)LSFis;`UW^}d5(Y%c8)(WtJUP7SRdma(W&C$AsfTIN8y+NM*eNk?ZR+2l}WfaW6#XAgL2dP=BdHi?U`1jtdP8Ar1X(AZ*=??;DE$Q>4(HGpu3 zpWm|*^a}xq-2VJo?czwsW5yZir$%jEE2_d7KXmk$GGl;mph#YU;R()wsy86<9zJ}S z#wNfJXg;(_xL8ctxxKx=@&^T-q z5m|~Hx)+h*2Y}P{TeiqQ{z^9vY^WKa6O}-LVph~t&4ac)uK1jajd#px{a7kf*A#Pm|xisf>R6=g+7) z!F46_S4fkBZ!t;ma8opjM!45c&M9n}x*nLeB{E!qR~w3MHnP=wtCR2^Cc7f!*%JOVD?Uo&Wwo?l;~S8C>uj zXEVQ1UMIgDA0MCLY*PE;FsJFHS_Cpnjq0D!*Ue>Eut{$MqOc>?|1O6ajo9S#tFY@<%ZBTUm+O-3aK9*f9 zx|xnCHdEGXU+lZ?nxU2!2sIP5jws7fL+xkskiNWx{LgE`E~D&(4LvD(To>D&b0=6T zMM_l9CV+s5d;0&@o-%l*j59XCii@xBq!sw^MGxI}Qu%`nW#s8t^A?hUbFD=y#-R36 zeda8nhiBaWIeybaDMy-@GWTDv88*!f#AVFdIia3>HwbEWhJga;4cV$8q@n^3?lV#z zl1><^&Chz;2z|0xv?wTH5Wcz2X?O|*lCt=G8fg?}^b5XU2%$D%0sweDQM>FA0iREj zJ4bHuJt(KZfUi6z#t#``GuW}+*nTu21ZaQ9`%4&n0MCXqM2=a|&;R`S6MfCmy5uM7 z-9Gow#AzfMh9pX2>d2ehw(@5Kp-g6e06lGg&wo5_HRP3%i6=cVVjvMPB z*1or)un0R^-*E#EwjZLzVRJQA-?G3lpqg_S3V%pFZmq`3wE(`RF`2(xSK;T73bTNn z;wDhXbx7(oOQc`F-FY8?^v|lxNRYJ59HDGxX3&c(`RLC8dMdwH*5dboz zfJfL-BKe|xe8n-t_^g7k7nbBJ7H}H7FxXqa4my-bRY@VyOhBC|x?{Ese2P-puyGjN z6LpOG4yelG&Ay6*4wy4BT0gN36Jxrz=y6~loxmgj)XMJ>LcF>ms>+6?anYhh&Ts~7 zM4$xNCQ|(vu5!foDw70p zrC{FuIGHSQuZ|(F>?wI4BUyQkjhzvu_0-!E7zN6>ej+85k-J=}VL?D!&;$#4S*HV}cG+OzXp{zI(d$s(xwx+9&{ga}Sy}KbE7U^UqncKcF)hX9%PX{8Pm&gvT${W!UWtUi||i zSmTJ>cqHHjZTfG!xqT|A`ls6BzcSB&#oumv9;&lx49Vb8rzDF@d&@^ih!lQ$UE6;l z$IWBR{`bLw=u4Vj>I1Hs=qb9coy4RhIBL#WT3$ia&K`YEr1X3r9uAI~qULqi6hSVgPyy(&JNsRvFA*BE2|4})^^_v zkf|Y~dgN_u_6w>xl#jC*0yJJl-++SlFlxAY%eOI5$C1exj8a0Zo6q136j@h!e!}tA z(9v;|tZ47kY9@L*t^ZE{Sm&gY`kY1}cS_ z4HG%uF7eI2Yua4CpIw6YxKZ$6&x>@bAqQ^}@`PgI2Iz1h^Saf~;sJn{9oFrO$m~G5CN{P13h2!27W=j!+igbWo5yG~-MO<5MKEq^8zd4Dm#FmS zDMpP_2DuExY|y@~;R1bE(pUDYXSb%X=59h=by+$~v$vCuSq2HrnOL&MmP<}ke3WneOkl(PN zXb(nUUdF@Qz{@)i-S|7up^Mh7^S~AApzJ|&o{}Jl38rEDbwRYfAX#GFtJ|39(F7IC z3}&#a>L5?t-!&Iwwxn3B&Zl>Xrp*>Cne1VT(5-Jo!pLN{jfjE`3f%hiXskws;fh)F zSB87$z>~`4K?uQHXa1kobeH)Hu~AgS_fUYZj3HM|3tp9K)w}yC;+EfB!LxO^_o5#} zQJ?Vi1n@aQjYvGYTOdB=0cu7lj{}l^p@-If8YPgawxhGChWHbZvMtn(iYKR*v) z?jm=0eKq_33;>cP(1uv+-i(1eS;K%AEkR|o_h8T(R1+Yr{d`p3k2UMbX+IFRd8%BC zwv4+GQmVx;-vsRd-cVri+Op+VeoIQY+rMq+kpT7k6KGb#Pi0-1>Cp_m7vC)h6ur<{ zZ2R&UFE6^n4C?vvP}#Y!EN;wS6{(`Fo6DbGGGD!#163R@4oLC}iB^gl;kS7v?RZ7s zkhjocH#KBkZO7OxdEc$`p#cPGxdC<98gc74fRG}S?KeM$^jB{ruiV#2l?@PzMn=6a z?489@h_p1+jwT5|QH$RQbbKL1xD*Bu?B7w20>B#Gby`l7mNkKGsHzUZUqGr+sLaVx zVSxvQ-^h7t8G9maF~NEO&Ai8^O`BGRS%_mm6x1+|qGXG(N@s|V7Yx z;gRU2y`~=#y#b(2BOW0=edk3-M;h$}1YtQi)N+RysAX=VVxma~^4eQq|C1VSm^PUe zh91RR`W-RY>sajdg8$ci7~zLUMnWNuCB~Lora&nDkY>x}MWK-%gZCVx_TtO`pe@RF zo-&OTub*PnRc;!~1&g-rP%mF5^=DWYF={3dOHdk#%7FmPbO8cCIUF_sZ)lBo(am{bqX_orv$!3!E3H@gy%wM<)lynO|zrG|50FLjLX50xe`VoTa zpfmC=T>?NP2k+bhixTCRp|@ETummU4ga-~sHC zEmLVukU{A~cSQIQ_>qWV))(qM?}&Lg3w$S|ik8io9UBS~R&KNyvzQ0lPEY-HZ-&~| zfp3+TtojjPRmcYvI63irV<@lwmjO{KuelBmBaxH=fk5!PJDN9@*VoR{IL$Rd9xONs zfg!`%4t$@MzRWW*DIG8REmy8teXLaMgj?D7e2{b2%{9Pxwtez zOr-uY*lwVeJ{H3K();6@_dP-c^>2JX`|J4i)2UhJr_1rb3b^$-#_P*McAF_j^s4}A?}E*x#%o^5r3Zw6kSHd%#Q z#)VY#&qCle!_5(LgdqJ6@ogw)6t$%{&F@P%8w`|p7sgAX!##UU>e*AYulVU71|IHE z{msKeM+$RNUdFCM8gksO({SUg=mE@l15-6UG&EFEd0-m=87_OS=22|$wp~rES z5qug@*3vJ5vYUJ=0*^<-1=7+?)Qtcf(?6i<*6J8@~g3+PJ%4?{V!6nfc zVTJ^?doNtr0a(iCT}06<6EQ7qUnwkpy*YoyCi+46JZhiRQcPB$wT)!MDie{oIJ`(V z$1MwqC$`S_^=oNRtFa|FT#G!xB4hPII?^8l4(1UgewZBQaqRbx9hr5OBav~jKZ8FG zt*?Q%^OJZz$j=$D+U&>svH<)BpKcb++p`oN?9`LJqz!WcL@opd4rFvxoQJcH*+p3m z-o@DGf5+Fg%EW7}$dJNPQFozmp`{B4etT`&b{G!g5;b%Yr_Whf>_#CvKN5moAyjeK zQRs)_=^0i$Wuqn>WM(4&q*{;0;T=c&=3BzvP?&ZqfZV^ zp(#|VpYbY)gFzTwgd$TGAj@<7a!~GTy^H)OQT|Jfe&3G43=G?rqleZyT453m{IRo9 z=SqhXCJ_!B4aMWwyGH2k%w(gBO%*A zMJ*xgE$CkzK*9yZtK2bSlq;HVDX-e2A+5*QlV`KBl(*0Yq}JKbg^+4Lo_jpp^;cgj zJ7JO?^6U+Cz9G0G((ME6c)Xrf{j5XpE#u31>b-Dx(0f$X-v!Om8~fb2Hg+5 zhYDZxLdEe2FR&nuw>e%6^9)FQd-2Yvm`??HmCZ{T91^jP-Z3PJ`S%3_tYWZV?%=s% zs?^+#p=>|VbR%ZZf?nVq^0j+q{&IzyPV{(as%8Q{yo~WmkDyj3ty<-#bs+GPa!yb& zX)I5Zy|!rE+lwRcxudNJ#oIRg^n?}b#szi0+EEULPDhN5#ADV$ro}YaACA9IhWJZ6 z_O2KDRT~|Z8dW8%;1MEb8lV}M*gD9qQ1he3DwMrSNAu)dt0oPcsgQPe5o?+883;XL zi9Z;;6<7`{BZyN0*vHV2LwLB6)Ul|6XEV^7mlx3R&Z)b4wLyijqeJ2OgS(jSjB0eR zu>+Wa8O|56?C@)$p2y)eQ;hTFc!u^t1DvBxKW_FOoF$ql46WbNOz30dLZc5cY`)9x2G0r1@Fn%Fu#89rkzHiIzsOFgShzL=I&}n;05r zbMw}%`(QZRuL+ygs_{VMVn#!w>MWPScZ@bG@_of>TgOY_ELEHMv@$+7^!-5tVADwx zAGy^N!s~P~s6qFLruuzf*F0eR&r$sgLC&;qgNq4UO&o-b`V~vjDC%YNSyCT^$cY z^&6@hy*)Nnz`vRJ2E&%G;H_`tlxG3(@;-Pst76))LKUChmhu-E?za;a6H`Z`-N=OF z9>qI~Ol@_$4VA~0A4+heHRNy~_x&XaA5>5VRM7gUKcHXN0c`nkjtaEw0Ec5UOvKED zho-EDSid+yFu);(@@1+bTDuhHF7|)cx0I^f`zbFSe#O<58Z+TQ(k*}Y>ti0oPPfe0 zpwSI9QlPK5gMNQ75|RZP2IY5>HVk>=hc5i?1>vGG8vm`1OVOq63m zxyUD5<)m*KCbx}Gt7;j$y0%x)sF={ILpdyOiBtxzL&H8;CJ+=s?|ezhZ#F4iX!>HujvX|glGF}RyYhkd4o0oDuHX;7WcJL~nJGa* zL0(*k_c)CY9sdoSWV3|C2_X2JxwsB6d6-M;l>tjUJLE%mL73=&iCZzS5MKZfV?JFL zA}86Dr>ut%SHlQnabv<)2aqF~M3;t=yR*DAVqp`ZaoMjJC13*Uiu<1{N z6WoG!Cy|`~%mH+J5oojf!C)6aL8gZQK;DD__F=rT#sB=X8S%&m>AL_CN*}FvIWi6V zI(HP;`V#Y3?>O}K-8*mT;?2GI-S>Fl%mE-t`p_U-!qBPy%079p=d0$-g==F}7J_ym zl8{$DL53~?QRqGmC-|XrMFR7ueTwppk$|S|0+;d)qAo9#2Zg8vBftz5xXW2_&YnM{ zY0NRZILcG3nOb%GQr!`7*b2CSY}gXyy|_YF+ZKyu1v+ikNpCik4;IoxV4u1Rw6-4F zj|2oQdfC=Jwbc!MqGwD3j))kSo}tV$o&;SqTtk?z8F(B?j1v*%KD5X;p>V)&Zu)4c zf4;dK&Uz|EK-%7>1%sM^6KjZQOQ^RsjA+&s^9_?L?$R>`1J8@3;RQI4CpX`VqX|jO+I$X)2IBrxfg-IqT z#}+e9jg1><@gde0V04i_autrZ5X0?@rYrh<+KIsdwD|rI6&Bm2d*Q+hGA1xzS@tND zylT~|;uUjgO)@omS>pfYF|X-f$2J*-xa^b{OAT6*U0hwY_()N2KVLs^{Xf57uUZTe ziH-Fh3M5ycc(CoS2wblyFJB&D3*QrFd*lHt4M~{;fBJoB}}(Z#U8 z{Fj1d@v>(qM-RXqrm6-sdWLgl0jpKl<7!)2R@{8LQFD7jbLe{G#tqOAh1g0qO=r!9 z2`1v?6%aTKwx#o@zqaUq)A@&C0`Ssb^vWRyPY)dhv2qeqY zU0qd~{;Sw9wI){C$zMAes=2bJVISp1m=vf6$T@~BQvG);9w7OJPow8w)R5iDvi*C6lylnga5QoW@Lb#mKMTXNj>$J&ZHv)+Y9M)}%fwNkIcO^}^xfAmrHaPr}R_48rj?^{F(+ zFu{F=KOqYa1ug{CJQuOd+fpABHP+%Qo1x6ziwr}?Kk73uPKb21M3j)#MM_GG=OlCk zjlI3kJhd@Ud#iS)8;bR21VD|`fW-F|j#~af_51NlYjg8elsr?ND@WNqR*Ox=bas4G z*SGEHMVUAFfBEj1MmRa?S!(9$h#D{0v(m@wp7OXj2>9oB^ zTl;mgibveAe6tef_=E9rK{n|A*$DnDOU zbr7S+7A9-BqtXrqPFC>yC16+r#W|D%XEQHgzRMrl{{EA3cqHHGV8=A_6%PaGCyRga zmuIc^80U2)7G6ry$*M&5Ru|tATtTlC3-%P=!xQb-QV6Nn&<$bu?qVb-gG(X;n-h&+ zu`5dEpq{w_2?b5S$IrKCIH=SMzdM2161xGSV7U{i@1Q;%%%vlTh z1QLm=Z@y3cB`t&ev?~8D|0o-Q$EH@zHX_?Ay{bPXXV)6X^Tl1Swus#T%=zkG^B){Ob?Q-xNU_o|h*8h}53^^_ z{!rc8+39bjhM1pFTt4E>b@;7eH38E23CeTkC-C~vRk+S3iKNgoki@b4V=h05R^9}C z(d4mTW&77{z$AZ~`h{#p=t&TZ%3gUNUL$JFRi-03x1pB+TwlDFKe9DKH5!6S4{D{>RQU}# z?&VcF<0jUv8{Kz$tLD#5Z&Doh1iR(dXYKyQv*+Bo=XF9UYilak@+;Jw)Ruk)E{_4Y zKTF)xdxgHlCA9)tbXqPYRU>yi1u$*nEluL0k!W@NSa`ns^GFwKTJ)x;6pr^OlLYqfv{g z8wZ_ym2b|$GxD9G;LKaHuo(Cn_*tf9i8=`>;Ihv;b>XB_!& z^sFvYQ!&_Ewbbg_WI0fkhI0rlr3or+Va3|cdbIm0&5FhtQ&!M3g3CpLst_}Q2SLiZ zN5J86@`MVm&WuTgXNTZHhh*NaODCB6I-&-npDxX&jZ}a6FJ$(Ly+sC4k zX(Q#IElo5XGI&jG9mkJam8{xvZashzgLA@EJLKJH`2p=S_fIN3P zg4MJ~my$F23d4^?4c@u5YeI|;tAd=wU-0q?z*s1)-oVPT@d_HL8WY78JI(!63>2*mx-!QGkV3n`o~P2FeJ{0cjb$O)HbqL(UgPjkJWL)>;L- zG0_c)ChIC!N8@#Ez-=O;gJP5Zf$h0sJnrr>03*zDDRV=PV4*(V2*2g_!#_^PaeJom ziv2RkwJDy=vwv4gn~W6-$Bd<|*;WFqtQ}38_T6;tcf}Cjw|~zm@{3!W0rLnc>u|L? zfOm+C?|+bjQ;!1Z`512D3nHBLRXta7dF}2ej8w19LT4v+yt8FO9q~$eip;_*(1!YQW+$@1G8f zbwF#32@UVj$~;Zxxgvu0hZAS${e29D)-@d`I#A)z&^V5z}4v@D_4NvyJ? zFF`*Y8lx2$vr>qe!IceZ8ud@F#e^Ca0*2|jCVA=4YH-+sJ4@X>L=@!YvfzfD0x22_ z_l*jLM^PqWXvBuCTa}F+P)hgj`0R#4bp~q|jT^yh3;}XS@0Ca|l(OJN0*-1KKhQKV zX5-WNhvF8P31TcsN&<0V{BI{A(B5-*Ib1Z*Pd{TjZdricaSoknSoAE@ta9CCv{Ph= zLHU}Ctw$3@P-Zux1X0(}axC!WU&wq$&sdi~Oj9b-_;I_<=5A*161%Cb1M>H$6pJ<0-bbf-DJFW)Cv;bR~|8ry{~>T z85b^Ap@8(>0mB}8!)#1#@M5cYegN|naHI;XT2KyqcLV!Q`t3foxZoN@$Hv{pKnFBD zQW9PGa&M#+b?;B{QV$jihvbRTT75}r6bA;;aCKSpl~%ba@d?#U7pfD!VxZM^usr0g zYX6eVL(>qh-JFfdkIpn#1$Si7V(bpUDils8@{Ca?$=J>XO(!n%p?dGZk5C&<0X}dU zDOm^AC^^1q&}F~JQC>dJ*41PiZ^`ARS*_sCsSv^*F{Y`tC>*0rq|pTR$JNFdnECPb zp-n#xJ5zz~cjsl}O50h2kF~`h<~1Q;t`*kWeiO2 zWseW8gT?@x`p+&9E`2x~v=QjXazVWVK0e2Tarrb`6b&yj2it z3+XTgxv;c2q=}+$<2ZbC8k!K2Es3B1{$!{m1-uCzaaJIrP~f%ZzXve&Hy45no(cxo z^QBaXiSa+Yebe>8-%Pgwl_k*r;z8COWCn*AgNQ5|tqC*;e)(+>6yL^s4Z-`UKH3y> zZ_N$Tsg^*zpJWpeL$ZiQbacOR;96*0MZVbMao4{VeV4EX(DkzOXsk5A@ zVBxI$oMy!d83!xoETdE=5ys?%9m<^;$=qI@_}DUIpr!Bz=(f(2vA!x&kX?{`7Q2@V zI%7DfFzMhr4szh~kWH-6JrWHQ8L%U&r)IOT>_)b2#J}T?AEC$>slJic9EOIOm=$1T zF?FeF<9R-UU7jv9dK8NHnY5B|Ay8S6UKKRG4E8yWy|Oh~#S_3+awf&Z7Kt2O63IVF zI|mJpJ6T8p_MEe^kp&cpb6AA{B?ovv69(M14RR6)kQIZa)G@%GdtOhLQSR zeUvI?uVY)W8?XBME~As-AV&nKWaO@{nVK#(v=cL)kgyUjwwzQQ1F0h{MaUg6Q9Zj;X{ObAjRRY(##iE+`FW_-t5EWYIW?e!oxPn;_D=g3(8q z`u6D9vSP7FNMH(VgWSdstiuzA?j{RAgEcP;%Lc}rELy&NHjYoZI8<@Gz@e6xTjl)o zZ?yqys)Vh+h=^*PnbPdICAz?*Gx@V>+qds8D#7hUUnZeKcohQj9psZeGUtGQ6Xgv1 z6KSOJOuav5Cu*Gk^b8O>sc%rH&ZohJIOSlI^D-RO0tYHdfyj`9!4tG6X>>cDGo&rq z=(Ekrr7(`(ncmFrjSe7PYBAOst?~r~yXU|-TfW}%gE(yksX~!iYcuSYz_-Pv8nYa{ zR_wr5SvdC)NoN6A-h}~m7{3Sg&2kD&?-okGNB^5#mw4K2EN~FVPLXpEy(|{?!gKg0 z?A67z{#J^ZJXunYVNa-iWl`=@dkrxLR1tg9exJs%h!kg0qr+n7YMB9HItl#o%(taS z8y*6@M5nFM!FA|Raa!bBwDm4PV06IMG9rZh$QzsU$^2vDdauu0#Nx9y6WPXb z34JJ*aVXV%JQ~`V*!w0TA*N4JZ8QSemw{XHE^N@$w8Feerws$%H97nc)dNOn(c}!0 zPy9^J#i9UZ*9Y*N4lIRg|cLosRsG1{-S(BK6GtistKjuz!?s~9C4PZxXEsuECp-w^$npwjMMGc_&>u7cnG!!#g0dQXm!QOZvWVASQxM^-U zn0svgMXa!%Dn8EOr@|L`3MV}v!aV4Z3Fht-7S&SfUX}ADPfZJGBU#ar32o}wzyn&0 z{xpRaa!Us!N{gV90j!9(Zk)Z0wf2?6GE{1dSSjNIrGFd)wD$t+=g6&*D!+>E3?W*R zFBXNy84YQL{sOM8TNlv7f%g?9{0)NI!aSrvaV??4$~%;Fl}`%7a+n1fAnT$<43atC zJvIKwBeUnsNstQ?T(xfFRJ;r17lam-w&9haGwrC2G}*5*dRTQe{u?d@765j$J%|nW zq%S$OXwZM@a1qpXB^cpbTt49E64h;{EUg58oSxohR0x2;VZ70^v@@&E#K`37IBI*9 zM)Wz1i@%HpDFk$kxXndqF;Gi+duQ-3VQu+>v!-W**SgxqwRg0%9XE*sJ+9-_WHKYF zO7e1hx8$o33i9b9eKcpsf*=w15QKX>I;^@QnE zG}Cd`=yEz@J)&$1Vjw zxAr$=rytVF&EL&@#<7)ft8Rmxck=j*R~$H*La*F&kro?Z;&(XH{7wFDQkPau|2zEW zfWX;LT&R4SVIwXWRYh5c;fZ&v?5`U^68It0(IM8>9N$?sf#&#XR<0t=;4O>3;jx?@ zI`!rvyIIRx*3gCn&;Vc!YyNIWI@jcI0$g)@1rC|RxBb_jP>J_jAJ2ux@W1`3zVUaW j?|=Q(|Ehyb`X`v#`?F&@zvW9X_#-E)EEBiq)Rq4Yz+|_B literal 145567 zcmeFZcRZK<|2M3mK_!H;DxvIAW+-K)RJQEVkS&s}L>VEKG^{=%dz3x1BUusIBT zJ?`hxd47Mt-{bo0zW=%I>yPX6c$_&=pU-i;$LsZ6@5BG%dBvR+j1**KWIN9&$*GZ% zZP6ej+q`Gn7W{;&c6JqCBpu~-9Mx^FI^Ht5afM99#L>>m*3s&^=>exJHyp0p+8pH< z66BZQJz(zWXy+g)AYlDpe}mul#x(&>h0mApF5B&tE;*2qF{Km#Z7^gEWhUD|Ms`L{ zR>L{sSI;dS_4f6xleaqbPUalwCu7hIO#X7`-S6U5x129+Uo=yboy1FSW?NYIbQR1e zc(>`&QO}ho>>{t30g;)B~a?{+VZI^1a1AU)E3p`0UY{;`7$O};+=R_g!p zN_?%|xncMJ_(Ad5vx^7W{`;kNcRSsGf5*lq@_&C#49EYTVd1`b@+28Mjeo!DQ@^VH zyLVsgH2FJKs{6`)DDZ#H$Jqr1cb{09+$4GH_YMI8fsU_V%bJ>YfB*j7?#7K%758=u znKvCs(aGFcR#rydAmYAY7umBp`h-`>s6LJV=Cri5hstplBhO?!3hE@Jq@*b5!uM*X z>-7wU8+kU@F2<5b84e@o&pj6=AMeV&9}+^vt(ml$mX`L}vu81;RRIhAcd44vjZUN( zm9lQ%zFktn){}g-s8xDpWyO(|Y6B}PtMA%#tKOnZf_>jVy%b@zG&S9*sHk|%x^vIt z!xyTDKZc70ONfi_eRuWKsbsCxi-~F+&tAT)c>kVEOG|5VcDC|xw7d6y&xH*aqGj3! zj{jMEChe9}7pzXEGVd`s2qQF zP+NXUDI04~&kla<=jYdZ@CLC$xNqL4+W%C{qE%_v$z;X0B+X&3wZ%815ttNO7WKU| z-#_sP1{b@{(~HYUOB>i&S|;BpDlBw+HAoZt^JcPJieYhqMuYA3>s&^eroQJJA{BcY zlVcQd)#Nja%gbZGbh-IVrcjVc$S;V z$DMw4sPgg%)!lvaWDha;>&t!X%E$M(UeM38+wkdy$Y~`dzkVGX8-7WrUw)gm?lPA$ z!Z)6)f3zPkiYSMhJk^tskk~6^>Z8E3T=#Br(#~f` zRxr$`bbXBp<98<{BxKay!1|5mUIhgO?xDKIvNG9`5m7nyvUZ}ShqdkRUj~+iJuj%gYHny<|t_j)C!*TiB4T+M4LCRz1 zO=LS6j~$s5p(V;%d{+1eqAL}rD+Dpd-8lQOA@cag#^@6QjSUT2j98I}NZ zr%_oxZuoHcLKLs0*BP5d@{=-7zjT^2z1CNAJ2R{Dhr{_tzElR#@q9b6zVdDb!#o`k z!5n9Xr89jQ*XN-cd7i^CKsmCu2w#w6vbO&;9yJnpvIxDB$a!7#~(q#718)&#ShMrk%SH&1Y^M`4zck$EP3 zg<@@O*|xjbjR9>z!2e-eXQtWKM~@zD9#7Uz=U+5dZvIyLSm@?Yjn;Id{ad!wQI_xA zv7;&5Qk&PHK*P~WUtj->k50;|Q>QY_YIeE*Y120U9KZ8tYvNtyFz&L});*k@oOX70 zC+}|ADd;+D9wBbOF+V@wV&p+!V8!~{{0XAsV%#z^GH&0wvjNNeVywc>EUT|H(H?XA z))t1Snyph>^F5YDaToWKG?UBmmN_{&=FKE>;{ zG(+K*E))t@Y1boBO#Ai?6i$@T8mNRldZdS z^vRPa&)i#TP|ap2ef(H`yCTU=+HI~9eNlroBq&JU%8D1gD#N5=OKY+=6ZT*@82J?u;47^I=hY7@+9d)hfWXiJ*4N;^&zLEKhpN@Lw$W^RUm`? z)Sx`Gl=E(ET#Z7_&cZZRDR_~v`})7%AEPS`uaU8S6fldwyBm(Elip1 z*e5KPm8PMlMuP>un})_4|IXw~$|ap#o5+?=mMRev4qHy&+aY`I+yk{`)XB-w=0jF} zB`lah`C`{U0n=(vR%Sn6P!-c!?sAwq{8jq#fvbEJ^`SdoH z<4(p4lLN}p(k3P*zLK}>KXk?Omrf1UB(q#IH}@;MyVVSz>uokQHI)Ex&@zKjrC37I zRBF0vX&JbSMRJ?P_VLba%A;57FG`)b`SZyr!}iMWR_$pl*dv>3V{zrL?1!q$85D^w z&HitfHo@eR#BSDq`Xq$DrRJAlH#<8!xi}Kdlbw3$odTC?mQ9aj98(*1zWLX;SNyBC z6)jeK0J0fhySDZ8>C+t@9Wf}4LPFiHzPFbZkc2N& zr#;uqf}vrfN6`zu+U>)IH2HyV2#aQ1X&L@7I+#>L~M zm?nx&e#0U^35W0f0F2lb<$z&A#_!2wPgQ;VNJ;cgwFvYgd6Wye$6TgthYmHzDpIBB z=c^4l=$E+f!)?>Ct4{|82A*DDTWw3b-Q3z!cq`1pRRbGNsa+GdLTr;b%{@f1>nrh~ z<1wE8g>eK@Q!m(39kSnOFgrC(Ki?alMbQbY$ECBAw3CO2r+F@ywiV@)VO4ta&mTG# z0j;iYzk3Rm|7ccfg+F`oB7oEH55{uf{(g$iO`A5!H(nqHo|97pJI7DC0}bhrE_(rn zCnGzX&)1K__S=V0WwCN}WpDL2=QlX3TP=*+R$9JWQqXQ`X_;(1v3@{ChE*+b4+^~V z>?k#g!#&_#5{5tT#;|f`gxh3)Spcab+p=8&0JW*a!>!1DQ7thV1%jx2sT5}!?!9^Q z#&|h4cYndkmx9nxYAI>yfuW(hu77@(_4Kf~x|Xk#`#*S4S?)_oKJK+Lb$@iHwVRvU z!oorf)^37Mrs=Dpk31i2upo7_Ey{X(4YT@CVjn(#era?j;l@BYdG-ma6DLkYMx5pK zrz8WzP;M-A{NXL*v9xW;12Egp!Qn10$D%zg2<`U>%ly<-8A=Cv*B*;Q{QSExGYl#* zmvnUA0|lY8d7ItHXEyR&sHlH`lxr2x6$|)#(?dWOVqMkZI+% zDhUpn?V?s47hkh)BqLgaVhF2mVqzj`{6pxW(f0Icw#2vB8d-@=&1EljwSI5&_#4L` z9hqhUjU}k%?=b}XJ!V5kE#7>6ca7nPJ~6pjpeXM@eNssr|B|Xl8X{nF=i1!FmdWqe zo_mo;bwq@5YtpPnsEfQw%EP>l_Ei4mW~$rW0HMqaJ916#i0QEomkM#ri36J0-%EKE z6cl89VWr4c8jxE12UEYOsOSQU#MxsqDpD@fuRC%M7T%g*Md91MXOGYF{KO$XK9wJN zAUgz>1;EoA;|uK7{Pa?+rr_q-B(~D!%a{9c#d&CY-goa()6?HWjd5Qb4)?Z4S7S)o z<1*QQr^Ufv&Ktywjgxaru5Dj6_7OD$!~JK^_A*O4ZZmS9A_s8G4#>rV+(t!JgBrYh z-#)H_PZv#1N!2`@y&~wsI{7y^Fun&|i>$kHymL^ijeCn+GIDaXj(f!^hh^9gDd-nE z?bJ-t;8IHoWDrnrbj*tJ=*+tA7Zg+lv_K8GiaD<6=+H^LU}&&g<&vJ>Q2q0x#0R3F z6Lq6`v!bDQ-jS#L21$;ey#_1&(YWP9N62qu_=hl!%C#M*@9#3{e}AVV+mbTY^1rpO46LY}l{(WGe7cU0A)|aEr4d_&)Jy-T(Wa+a)SdP#Z242nWx!TiZfsdO4qj1Ftt^AK+#xel(Y@LqkIj*WSS_r*Yi}Ok%rwdwWTJEx;$$EpcZ}%*mTK9e_=i8t{JW<-Ix^ySj z7R*AwyX&FC+XH0Av42KeBjkrT_TkBaimBy^QU*6+d38SBtoPvcYNJ#3_V%M~$>GLT zZ|>jU3a-sL8XXbAE7{mpD{*st(K6oeQ=&ygyaNpgL1B>m4}VNZf|ATa|E6QMz51CR zkIr#cVCROFGz)0i%15HJ9XxU*;k}GmO>h*`%0q*LgBh5JIPZ$8aoox?5leclbi*LP6OwAIf^yBFLxnEqCRNd|t#k{=pA^>2tx7f}8cejL;^CX$7s;bi2v+8;wz|f}>qh!Gn?FTEj zSIvOi9<%A*i_N3}@HH?#9)x-1sYSO;5c4@ms(BIPWP2F>-Wz0nL!~?P`(N6ltj2F(O0SG~T>*RN7Ohr#C_OY);WD@xyL*s&NOe3)Qz$Nv!yxlNL^NR>4; z;dbl=N9_mYLk|l2H=2-b-Z2Z>F zv!3u|hESog)RXb{Z8_>ybFtekvfCQmRgXB%Zr&aQs=SL3HZZo?!Dp<_9>H*FJpfl{ zW@g^(JFID)_{@mFx3qx;Yxx33CHjdnpMAE|I_9*6KMQ6#Zi+ez(IQmMZMZJ%tkiR1 zs}8|9Iz>otc`h@YR~sS{HGHJS#2B!8??HbsKX{9Pbh3aoQ$O2y;^rvmxcg^+d`>VZ z%o)wYH-0(Su3h74dsQ3z^IG(Y?L4B_Wc|%lzO%Ejk#2Pr&ivVf#Y~#_N!pkvX4Adf zdjAPesc$#(2dDUdetmZsQsC=p=QH%N?cd@D6!&}hfL16qtzn}KZrcuVk&avAWUN9^ z#^qWXSy|hDl~d#6)-81}&;y2M8czsl2$SDdOWjIlck5OS`v1W1-)T1Z`n+H_LAckbLV7qTS-o_7tMs}u6 zK0C4l*BrOm>grWaQf%zhMCrOI^vH#Up%BerkGWsZ&DEM48!u#)c38x$=&fEq-S+bi z%2t?JWpLt)!=;aw@<+aXJF>R=>+4Y{uP50;k4OXiaY%mo{8_#E@Ol8RCfZ=!jY5Hb zmA4L{ws9{sHuSO|sC>k_zA_WinF&?u)csv7o6o#M=^KjnT9XAaqF^@4wJ~UR(oBx6 z)H)76&XanZ`sEDBnD;k3S%;(mf?oEjoJ$~NpfEv*&T*z6T-4Omoc^VDVJjTIG9AOjud#Sw&etvj)QI~ z8~K@c=ViuRz`Xs2d}JP8dwJWgUG!NDSxNVZ(UJalbi8d#SGBTMYxa9pf_|J8XY3*5 zlTy#sC!J6y1tB-k1e%}@Xq=DHZ!w~*zO{hU$$v+uyb(u(~H0H zPSowO2l#KEBhYD+R)=LrO^u@CuX{V0P=1rj^r!tjruFdTmNUA0P+1e840ccT(W~n zbO&q^ZziRX{U`VncT--0B18=MppbQE*2dgxE!htr?nb@b3L&NZ?Uj$4yBMxqxnlC? zXWM8?T;S-=Dv+8zKUTn!ws!aQOrZ)50K$2HQ$Bxw*RK6C51@g_a)(h=2e#eh;?d}N z5PkGoFq;e86S`n?!M=OnrJHpI}rfgMLNn?(E{?AN&V6W|ms-(-yLkhDcg=b+A8Q&}6PhM}SkRAsrG^ zB5Ga!Ovp@GDv<^x0TMIC&uFs2+FzKTS@8U#XAoqH*|^K-n)-Rx`;$c=>94u2rbtxG#uYbUxA_jEpx{3J@i z&`7k`J;;bqwoR<93D>~K*EdnW%+%D>8<3K2aV0Je%r_%9m%&Oe2T};u7i}1iwgM>E z9rBlboZ%nw*LHU2*>8IPa)}&3j_`&KJg%JtR&TzH9;3P0H~ow>3K;Z`-2|mpbGvav zxWMtpMm;?}aX>rZPHt_gAXe#urjckNvn` z&bemu1MhN1zuK^211Z@em{~%tU&|BUjCN-82n!4Ev6gn7-JPPe;WoO+DgJP?rWo0P z#!}#ullb>AsoE#o%trvAl)WrzXlVXA1aiV;I|vzRju?;kcQ$!DL9NiwzaekUX1cbr zNUU}>$b>yjac3Erk2{@9^nfe2FdS~w50eZVkh?hD1teoLyuId`U!b&Hs!yIj_s7<* z`1ENf+Mx5wLZ)%Dh=+$qie7Ho`}UKB4#N7c4&tAgDHOc_L~y2AO;Y!F9xkpkBfc$~ z;|$T!oW7ACDO7q-<9(T>T{W&`{w#l_9Jh^0>ue)`mAUY~O5UDk{6wV4X46a7%g;pfpYpH)&idFqsRUFIQ$ z2PaV*DLrh!+?ZwD_k!NgwVIw$QPI3~ft>8AfYF+v_5AS9pA?v`DwP$Ro&uAJ-ePX; ztDr_y!q0a^doCriB@PY^sas7|)znxi(=Ouy8V1?#Cb0g5dxuG;6!V@|ro?ki_dC;KNRs2CU+9DlTs z6Lx2(_3te^_TC2lI%Qr9bW2Bf9ngh~OY)iRy1C{(QZpGdDN4ZQHh!>4wFv zb0UEIzrMaB_I$JVoXd-2*Ea&bK>ocj9uXWI+)?b7J4$bi6*T*Mh=Rx_kZ!%lq9r83 z&6_v(r)bDYD8~$I zYirwf?5G^738CJ%&p#{c*j`4)^3G01@N@H)*V|D`Uw^i}E`Yub$yOG=e40^dFd78G zq`A1b9@btaJ_qhsQBl!FSfa!b?Dty9RH*fR3(RB;MmYK%n>Eh;XyAMX^2 zIDTsv)TJ8W8@jGS=)h=)8z3m3#99=@a7L!{^6(^Srw3yyxAODzGalZ*6@~|f;>m)A zxU{r%)rb+Cjmc)k{jZElJpoy-`d5d(K{6n~(sKTH&*c0(q~^^4ZN`?Clou{sAPib) zH@?sy6&$|69D=7HZr4w?i&e@G4)H-rNk(v8)LKQbAi^+$Q1@ESG7u0j)aJ}L4g>#S zVki9jN_a5v)ClX~zg`xCuTK5fH*#{XxBR2t{O3pifBpK3j~pAd)W(e)m5=ihglgm8 z%hHI)YW3??>L_w?xHAM3J*dxfc_)zZDPVKeSEtDL?%n&ozJ3Q-KGD31WolcpxCNZH zYG$NSvq8jb&AoYVd3m|xvZ=hBqT&$n(=&+SFw3{0l129DY~Q)F>QEVxzpB}@xhU_-<(zZkvPyiKUC(&pBiI5~{B+|gkx$jHKK))!x zygJ_}Z{%g)t4~2ep%y{>9mvp<&&7CockpPX5Fra}ubud4V&gP45W*HNX{%O>&K;N` z*QIX(#=;Z6vtiQ~!Z08lH1yqbDk>(nwjoq93uFcbj%8WbKWA8XiAs5{=nU%v74m1l zbOFB=clf@eWPQzz&~mU4eV3$m)57Dw^6}{_RJMOLWh-lI)nqNc2m6k0d2>Fp#XRc& zO9VuG=D6EjO5Iqg;Dqd%?PxLkZuOo<&3gLusol++@`!6pRWTU#!`Q@5CvQmoY3mbD zr5P~Oxrr=MBLJk|cy_co0Tk`m__&F&F&RvI_zOG1VJp#c?0>XyWKZ0Y^j&0fn*MeM zhWobay>JS4jo4%IQjbLcy9@AAenFNXR&Z!s}o@ol5k~TpV6+9lO4!- z)Cob{dD7L{84%1f3OD|UWM7fXjNJ15d-v!+S~kqg_jy?|7nYWmR?ndI){THWih&0m zw~dzZ_y%p1dqnr`W&iH??@tbi|M~N0*vR8gnAGv(dSmlLsQ5SFYeW`p?Od4avZWh= zF{=_f;kxN;hg(ithQHT|Iu zO3ctZi?*bV>2XDh@L9gW)8Zcan(y$Y8!mM6yvu0QtLbmwE~oZ|5E}b}L))D6ku$Ge zz0x%baS6ey`)G@V!VMcuuS3T?{J0t{FYt$dByjpXLJ`vw@$p%!Ojt}p&81XuW2do( zKSIP@!FZA!(5lp9pGC=da7%guLCL$O4r^X_-$DcnnDqZvh~Z8U6G3nQc6RnBPDkXQ zzI>THBXLD|AL1YZ@`3aRd3erRuY-lZ(oEI@+@zCUgMdOb^2X?wF9X+CC%i`Cd_xSW z9wIZYe zkfM%^S996~(JL(R?2;6$xISAf9eAsmH|B!vm{w3MPrW&K?1}=U7-awz7JFWg3IG zA-fG$P}b0JUeFWBhcWURwmxYbYUF2OQ{tTltNuC|VS1psgoZpSgw@=dq)Car=37{J ze8-L*%A(>dLZYJlEemKKfxVN$A|mqe9!)GQdBmNdWIkY()*W&{;)*~ju#HG{G6)@1 zqxp(NIbcyS+LpaOi!}kqB4^O0_fus#flI_2ZD*>Sv+ zj&OHJr-09i{f#c+ZHAb@Bm_E?%>ExgE`1wI$l4?Fo5*N^O?f-PnruzEM2qt2Z6;t? zL@1Ra=k0WLb(11^8X+@G0S%fn=a!b9AOzu{<#4)&h6YTSZ+g0b3@pg7iA$!YY@RDq zimBh{CMPe}8#J9(FIb5DnrDBA5UXFD6BQGKz$!9Xwsrsewzll|;~xUf_G%e^{rXjT z9K}}LdGY{mD8>9J9B=>>at9)1GJH+n)70xeqlm?3Tic0Ka_6zhHQfJdqof;_$n-#v z)UY#xUw*X+{h**dJJAO z?jWY@_^q^>2Z}^OL|0uMg?jwCGR$x&nj2xB2@BI4Jb2K&JuPzd8DOzx77ZI8AHf3E zx_ty{$4c{p;0B=z)!>)ZFx)*l~bH-aaWXJqb|&3LZuE4i-s` zCeASBOib>^#+%@eQtk^EV;2cvqin!sE1^J?emZN5gdKYYPd3@2qA^B}U=>V5r)oBf z(>yKzAW3)gi(heTh=Nd5KSxpeqIA9GwSSB)p|q`jZ;0eEDss-0I(k%n*ah5knD9~; z5euoCmnublI1;qHwmfmx>cpZ}sxH^}Xfa#ES0jr{W9jl{zgrU@OC3I}=y>k@`QRel z4{4}m=_iluIz$HK4RGnP3&`$#Lt7@*C@0?w;C|uBT4ddxW7&yS9dOhn#x;BK7a9|4s}w!zZ+bG=7A+tkUM52E7NDo_8vME zyK(I1b*!#SC6Mo2V2}+1Nj(N@`-mXcFbcsmDq-!`9eWR~!zTHbnVIRGiK}^vPGhhJ zHd}|%$d?Xryt}GO-Ugc=ZS!eYSJy}I)?qA{M-agE`NNdYvZr~=+_mi~;MjlMDWwP$ z{{}`lAEqw)@b~FHuXTf=1ti^uf%Biv!$0}`;|Eu6R@M{bifSVxBZqNcPuIZ_N%n*> zLiNw)7nXnd@(M~Q?JOi+)ljn|0^7sj?ow3$`Q4+JH35@-vj;pp3Xh7jzpk>8xe)rp z5Tt_m)>5n@c~`2e(1#CW93M}i#`?ZEEFz+51Hk3ij*7`jO|9DBfs8~V^UD`6l3H(} zV|jbRGU3;#UknTPQ3h5gAA%MmzsxFeLvB{emv0yU)B4Q^)`2A`nQuYZ%a>zQ4G=qg z_)R6vwgOV1Pe5^Tv3cLyhZoPBKmQbO${qAriG3PE^r)1A5c%rPF0iz^%XQczaZbO% zkk2^UXr}XtiE01-^^143578C2(gtYii^wew17neduHQg5!$2=K(i&+M`nF>gBA#QC@uwN4{pM(@(4?Vadma)FE^{s95yKzDSF*4EaU zAo-^vO9dfIq#a>4%D-_1Nw(69Gb_Nrq;@0FR78&T`*px{34g2!X|!PMf9`C5gLOj6 z1S|@D^5nHa(=bR(`)2D-{dKb4d-v)jI0PE3lToC+dlVJL-|^EDP5w&G+T(brINos7 z&Nij}yrxFKg0=qP!-tC&mfd+m5X}UyH88s`4BUsOp#_Dv`JA%;F-VomQP$+d&SveYc%$L^qNTv;exjlvSFX zoNT^~X`;C4nW;-8ulXjwPhN}mfc(6m`f(8gF8EGsQ%$epdG0Q#@*d1bi zfXYN(JoP&(636{Fi&jD0-t969vOqwSK1*Ivv7)Ps3431}O)&J)BX3k{gl}lJ?qYs_ zd&4Heyz~J;*VFHFp9$)p@XpVdI3FQ?2jZsXF-0UN zslK5RGz&xumYpcKMncR~1kf&$@jkHkNcW!VOdmmWqh#Rc&CHwr#?gj4M=&5*8i8|q z*%oxdM~{+^pAF})0$|xZ4(Swd^EMHk`4kTuW*>M&tE3?S-`jw{{uFmEVDT2nK+R^5 z4i>Qar^eDtO_54M9&QVSNFOfscp_)Q!F;_Ib~>$fZgw`EH56Tv*36!7@Rbx`#l)<> zYm`x!xQbyDcN~2<=z{J@=oJHQjc|U zoBNeGeeVSf(;z0XNq~hRyw_@Zv>E#p@*p3|8Mx+<*hwhoTe1xWhjKXR7y2=Mn^}s>T zkKsFZY!8GY^58=Ip)D`OY}0Q1xtoR%Qa{iJLT_L4MJmTcOx z<2Lw;Q`z?!ME(%h*KM-7eAx#sA{%5U0GvuJc9i7X=mMr-JET&t|M!6%_Ki39g>%D( zkaczzCon!R$%FkT44V!M38_xL^9u>70kc0mdK?ZgvC42E&I^+VNhFdSZ4z;u!pSM8 z+8)V?uq%JP57qqP@89pC8q;B=$F8C65J4WuK$*s68>_xZ4y)d{@z%UBY4ejLC+pl` z*QHsmExVYHINLIbSooM!-186iM}7yRf%0f&MHsZ#Z}yTTaazFC?J1mObC0`1rHC9g zJ6T#<22}_4bZqqzoB#Cb(_a3LYIeicb=*v`FHHKDUdQA>-!0B-rR=-VS+p@ zVJ{iigK!EVIpgf+7PuyrlA4-{Qwn7xH*|Gh9FR!;m{<9q#7)>sY$Ba(i?GqT5F~E2 zT}#~NZOZ5`6L=6;_(aRQv!lZpOv<~>abRG8q&I`14paPh4n!?lj}U2QVAFQz2Uz5A zv?#l;6&0z4$+gJe_yOaESZ)1XZK!_L2z+Yx9#}&+2zx}%QQXE5Q`DmkVIyF6{KunK z2($V{&Z>^f0-A`>6RIL%hMHZTi1$vLA+7Bya-Q1g>FJqwYa&Xm5H1M%Lj2pby4*g` zq*nVOSijb(dkcjk1{^KVZope>h|ow83Ga`MVLV^Jp3_4!$$5zgom5s;9b{+U zj08SIDxJgZs47rzOkPDLy0@lHo91qKJhOl)O(h7|#rb%5bOVdaVHmPyu6Ys`Qx1c2pT zVp{_h!bLV3HqJ5R`!aVL^~OT3BGq1oI2<&H%mVjVZ)3U$#=F@q4kDxTWd8=j)&~M7 zoI{dcTG2QPO3lL5kfzNo&8LZ-(FScvJ_Mk4Qqk62SY9Sl*WO_pkXaN*-W#@&Tn*9+ zRN&m|QWGt4l*qu4nk8<^^M?~Lw15%$ufHiT7JIyiPiE2ZC^TsM)Q>fV4AM3Re*O2L zHg>q+Ad4;59T^*dqT8H#-oBOV8r#2nouz4ZsnlU)7jdoysBHV7IMe`0w0x4k0cdne z-17%r;Ef7de&G#jxI~Y#K?TWATPnogzYHrw(7N*&A&@mAD0UBWFWa$W8bggEz@Bt^ z;|$h`%26ImKi)u~NFU6{VH0Agabeq+por)W&v_L*0$TvNggIncTr~*Egk0o0>@K_S z_4|luJt4OtggBZ z!NEk-uX?2YTWK;_Ox>ox@nX`vcS(s1ye@Clb2cc2m>z-`6W#mPty|4Pa2`l~oxD^U zJ?`4sEMZAbDkaUumhJpdie(?3o0^_3M?t-qZJ`-vhIo6*0vgUuf7?IgMO+7>=2grP z&%z~VD$V2<6a?gy7Lx~ZCzU);v*?vVD!&r(AtD@!w9ybs7qVSX>t4P^o)o#!_9O9c z6}rZZI3N3S79Go9TU}jkh7c<&T-Dr6O;~GUA7^;=a?VW!d7eI9F>jMB01SaJ{7%JS zX6mBfKshj#;-9ebFlu(q`SZ#MG&x*f&q>7Vn&8KpS>az+8 zc(BtD{dZ&IJ2=fdSfzCAhElL?AhtZJyQ~!jo?cZ7vz9n|S0&2hLIsh&ITILLZMN2^ z$xv@GcX@hdMtyPohDu`6nFBd24@6gq{%B7CywQ%#L!Ju*Tf0lWjIPtkc@Fn# z@7}$OF=+deG6^M^ce?7@UE@-dNDSP*- z06OxhHwOTp@9FbvyA~Bs$K<)79&;M5BaBWHbXr>@Rp~8kSZjR9?ySU4(Uc=OR$slb z(I`3z>5giDFwmrhR5;Xjm0-fjLrW)hd1Tx4=vtfV(3Nv_rhiO`m3-*0PdQrrCOtg_Qg}eGV-q5+ zm&~qSy@xm1)_K?4TW&cpd+f Qe@ADn*EiIR1i7Dm(5EP+A7*cKB1pEtNGNW!2o} zjlJv|l_mC`$kqXx(2I-OUTN#X(#cXzD6&QZft1WwH};3>q~jGo(e~0L2_h<@aqM}= zKQWQ~_hFaV*`40$Xm0#v@aAU$+A^Ir>H>9^KgXqjI-o^BC|RPN)?60er|@9!D@`?d ztK~HKSxCgw;A~M#f|rRO$Ine0H)g#Uf)PN{(^+399Rlb_IuSeQmz<{o zoZcjgl(ejHc1%s-v@V7tfazbj&!?Zan}}R-o2MyTS*4ga|9HgjY4`|s-vRUE9!puH zVpngCcXF67e|vmEE|;gazFrQAlnkU*%Q72fmUjRc?^Zcd6i8@5K!e;-w-A!~;PB$* zO9J!JX1$4vGXX;(@f;*Oi6b-kU!@Bd7?_t4a@xH7Z5`^xGbxw13SyB*g@xsEvt9bq z){$KF`w~uFxJG333r@*n|NR>v=|36=IDoS5?oW3@KmaEJ!z>cM;RGL_uX#Wk$PI}{ zs4~EH>y|A6YYhxf>Ujc+UI*x?E*8D6gP?ls#YdbpP z{Hw8|+%ACP5<3uFk``epn$_wbSjlp>X&XDOV>P~jpy)sBK)kf#;la~;+=|b}snw$; zfJIS1K*Fk#+?tS1aLi+FqA$IbKU8TF*q0E_NP0nuYK2bLAW8HK&{L^d#)gOEMq`t% z-jQ4kY+hbk`g!>{BN+jKMuYOHTQ)cQq?G-NYq z`KOR+AkG%xE;m3}^Y*+DB_#{FMc%eEY-FqyU6j=2NwlB<*Tbl`NQE3n4C>J6VW-t= zSFe&Fx9AkRX7_jen;%JiLl#39`SeC$y|7bq z^282AN`L|>_`WnphgC?=r1ncF9L*3)OamKnJ`F2L0Y1k+t3Q}IaVp2wH2M6r?UnF1 zkmccqwo7UW!Sdg|di*E?D!fJ|%%~CpqWSv4{r$#Jtr zvO>gONma1|Y!a9RlrIQ>teXXtC4%S!MZse8PI)OPNXewsqZ(iRt4sS(dHex)gAM7LP z<>M5f3Dj?IxsP3h)E&oEX}bKYl#iG9Tvl)$qGMa&3}+8flE%hMrp3j@jWLyw15*3H z4?+vL2-O!Ml`@3T0n)fjIjoG0Z^K`I_nzrH&I5=e)YT9rO-YuKmF0sVD+<|eCZ^;Y zFEgQV$G{L~Eal9!JYNrJJAkz}RkTPbFtMIQoJpW`h15DXmwg4!1n?oH4I@UE1zHN? zkdi3Xa(&A;vAvfa2;KCGR61C==52)Y^$$tH!hD*cfGKNP`g*g zJx^h%j}9r98myB&jYP%=9S3pA<~1A-;!bRGbRFOD9>W%dlm{1w1bS674zLr)<}owG ziCT|w;$-%mrYDX^2ZUWBz@H@aB0)g+55qSeAm#twg@`nO2`8-Rg(+%xd2Fz?_MeeFB?@&8B>z1H&WxJj*TpNYf; zV7VYPhPXK_V-ofvjt<@hf`f2QoVo=EY}PC$7SyJHThe2QOnIuWl!@LcR@I8>3RYOv zu>SG-JPxGR>F%Bzo#Y5l5AeY#l>CU zgaqyrlm&EIjJ`|JjPwSBU5=2y``L(8ohQ^()RQSV4YP0Ab4uY@WTGEJI z!UAM56XTf0b$&ks2h++NftDxn?Or-MgggDDv(nH3h8W2E4qn3|okVH0eo!Zd$gZ|F zIy5Wl)%g6RF(d9Ui#>&$U9?ZHVc}bhZ5#g4S9$^fS*M&)Ad!Wss* z^s*m>N|SpFJeJB^H+naU57ay6+37Qdsk1 z$q_!eA96bf>X zrIoB+_ZBkhc0_1e&2I)pNAD*N#lc_=J9(MC8f_m8gRmOU9)H4?te4A=-zKO^?}3)X ztDB`Xl$zsMg~bTLhiJl^Yd{$lLRf=9Tla|HFK9bVWSB;|lS$A}YVK^>n&CWU8swQ< zg|pjq2rcnsTmbLP@K|;V8fur!q+j5;9U^4~5v{?=B>2BjWQqI$j;JK4#ZeG$+w9LaO2AIE? zaM|0SxtF6r+QBhLDDNQ7z7cM`z5N~BKI93%znVZEM|@t+vg=EnA#wWM?GF)(6&A=T z1at^{2%5y}*RS8Vw9wG;Xvq;LFh)l1WBJEo=qyGsPke;Bf|Ly*?c#VVH>-?$!opV^ z0h0a4{7=x&7X~T24`gSWBf;&9J zN`6@tmf>}5hPfJAQ=M^d1KRn+G>e=Pc?WNvy`t$-bX`S;WQ z*U={K_y8MsviGU^gX}*y8gh15e)Q4%P=7`?HtDX=+hHf!zR_>HyWKs>=8Ym1>c@5$ zHTd0o9=cPCO1J0 zYeVdguWYlqzhAbrny0y7jy+Dup36kXJxfSU*sOo}+x^Zome9v9{kNt!^q=f^)c1JH za99hg$%dJ`K##^|x zQ_@@4#HTlEMcLfnbmw#b*jEO+`Utw3hTcm7PwMwNCZ5=D@Uu}p>*M8o9hU-WR{5x! zpr#Yeua_?Nl#2i7quA8S8HtbjHr`DUOqX<;Ei_@gD{IM4R+yZuC_=CNV=a~1CDEvA zeJx4)@aY|7ld{7$%HyuCOOy6vJIIu3AMjZ%{MWjBO->*P<>w_)U{@DNMdnRk<3-8( ze+LdJj|=DTxMdXOBS+e58~%lZmi_Nm!ynCV!Frx3?Ig^D^In%-3R|uw{BAwJ8&=f7il1qAO=<*KdyR9=0%?BVyT zue2*T1jTYxC2g+!yuHsL&3G&>G;eX`07hXenbHB-qZ2|211&kn$~pft60QP5NbzV; zTz;!>RUo+eg4vqvrp7gM?Ga1a=)5O_DXYm$ZykPC7Oc4d>%w(;?pr_J@95Sa6-I@O#=NPuXRgFV37! zGhwVhwwf^Xg*skAE5Z1?54}kBtCth=&XLqXW@VJh19boO1g&B(>|DQu&BJv!G7cIG z+W4&c-_zV;A{Ut2^!YQx>_;qqrB?E1OkL$|bDPTSZAY}oZci@6n1($6PYv@U>|k!8 zmTs$+-+ffMHe8(FQniQ`QwcX=@KZ6flNAU)_QT~ z()`^uw*0%LSQWP%`DFA?O|SkJyY|gREgc!6VJKWN4A@up3_wyT^>@_!f0$r2F^@C=f2LY4eK9@+9ueG0F^i@ zYIHb#9=-gb61(}Al;_oEkebq_+y-%UX79wv$l=cR?rvtB>IMA}$72FWZDV^+c)H&i zVdozEyBHo8c#7{gI1!Pq+Lta#zUL=%TF!oBxsTSqj6}i@uZcJM&v%SWNL5{3KcG+P za_7kdUY!?=v8p%4cGwOFev#39l|FCd)!2!%j9JnmB6NgRMLd3pE(=->S0_jeK@R{8 z`(ZpUz~ZpS^9*2Mp0w5nWWia9tayRbX2@S{Awx*<^jIcB^9b1rWL%J~fn|u$Rdyj5^vQ9si`wI z_*1PbXbtj={$MKW$f5c5^QQM_#Z57`rmEcn#DN{W>M{?9+V5Ta812mbs^}-1gf#6n zTi^a)sk@!?J4|L6JXAk@O?yJp5a8xk&ho$+1)U6I**hMUgvJG6uPt@q+&M!2&49gC zSy>qamECz|fiTlG8tj*5uY)#{1N_Pv8^=_SXyIfw11QZVGQ>*_#-{Ml5~QtoW*2b; zCgApMvZ@*IcjB2;P-rEHhcn>7My%}JEs)IK)e*;Ug&>jP6$6Zj+_vkhp@XjB#a)T! zC@Qnl>Dhiu*N(TavA$36UdYv-_KDW@pB!30eO_70$|A?->__U1*_GY?RQ&zDohE)o zd76TpM+Z1;_{IYKZtoO4mU{<7W2{|DdE;Hn^iU_&@?Z1~?5Eiun7yvx9| zgI6iOXxv_oxnM1W{6g&D*&}an;Q=Z;uQo;n?K|4r%ISfp5txSgJ%eXQNb5bzk4fu-+NUUzS5X%=^zt`z?JAiM8nSwVPO>v)Hjto?T;m zPp)sbiDLMH<(m&$o^4y-`Qc2^eZ6J>gqcNAXUE7p+{GpGt3M83e5FhKT2bcIN~(I* z0tIm@h|s?4!fFLq!cT-geDO+keWBHHhh8y5zVIa>#W@=$;mdx5edMW<&vAUF97i}o zqT%I@9lHk-z2Qjr&nJTjko^#$nmXX+_n`l6KL-kVElk9dx ztIsI~;?W+&R(>~gD=Nw0`E*VVwf$u2Ox80tZYG>9qFeU_!8DJCYD1g^ApG=_#qUg! zH5Z>OaaHk6tg8O$y^OE%Z zDqmTY%9D}}|IRw*-5ExU!qfz(ijUEah@;nb5gQQV4heRbF0+fDt>$(>9z+7{~P$CwTt zFS)1xBB;(FzU+O9cZ>a@j{%d#{wJ)OsPb_Of4aFn`7c z3yY-c8F^$Zkqw9ZDhaKgon|{+F;c4{#BO30D96EPrh^wvNiCuJwF0Oqbk~O~`P;rq z?Bp^LyO?QsIBy|{X14a6v;Tdj6L)zJQ;I02-;aAEaQW*#rMz3aOcd!44fIMZoa^5C zIKU=ulH%V1txG3jl{ltLs=>2g?7=(t?p?W+d<^#j<-`YiC+KbFF`JJ|=8!kY^`5{Btjaloe>tlW>PD){*Y8oB6In9C^X&;hD|%uVW;dMSv3j zNBIxI*ZoRwQqYY#`LxHxk=74RU!Ld1_EzXg-&N5aSN&3fmQvi~7}?thULTgV4FU}- z0Y5i+aS!|lp=p$$%Mp(SBa2B-r$Rg(M;KvW?joKzgrkvRO9F5rpeQLKaMBF5#nk@? z>{;w0@b67{T4CTY^9#^+jaDV}e!>5TwD*q3x_$r0FO`ONp%N)2w@YYH*)18_E+nf+ z8D*3$(jtUXM#|nhWzR~8k{t~iLI{yPzt0oh_viEd{eFM_u1Ami-Ma_Z^}1fK^E}Su zIG*E}GwlmPWCugK_|1u&++V<@XYIOPzC2?5Mj&E@j->RGTi;Y3e(jyQ_G&lduP)oe zLx)c+D)~vb^}s%+)KJ|U*B%aCEq`!@N;5@n?P#CcWH6VWM%2c3zwE(Tb=qPZ!`r^c zdnp&Y)f^+lk_*N43m%a9Wk$Y7|gxP19CX^1`1QJ?MrXV{|yh99R&qC9&Ft-{e^ zfgY+ldj>sn*jakP%IbM(>G9vw;Gk1nfRC~6+rQrxtdDp4JE^=5#Gg>vvfQtnaptf+ z?~hyWIs#;#_1nEU`|67@{lsE1C8uhYnzIbQclIyHmq~gZ_2GJ}P`o>Z;!ZcydVOwP zq&I$yrRHn@N@r%v6`lF*(R|R6XHl*8LxtJPw)nGwtlw2$<6);}*tgZFPTeyxGFzxxmT>n?l%2i0 zy-*SsJO7MBojT5m;8Nl_iC5neCo!=L(-Hg^c;y7U^lDzN(AqhC!4<4>lSv!nAhLrkeofBKCcKE82R zXCGa@jE9}oBRYy*uK5<JCDwJ+|Gs;a+lG67qYOK3dgJOH#3LqX$HKwJ_*hb#P$OBP=pQ|&&>KxsM`6n095K6?NCJKC>wC6-clXU0cK$DQTO zUqq3vTi36$+Ri74 z6!bI{)4Gh0_&w2UbwQ)5K3Qu6kQ2#}K15g;x&k?Y7cO!)Npbd? zi92M8Y3!d^;`d*-h1zy;qydH;!Ed=h0A*uK&V-+6io!doLGk;qmw#MyR~kYsI+q0_ zzB{PA4P3hYm>k1w?u_NLVC+GIW^rKtni`jKd ztgLsyX#v#wu$(bs?`@QB$~pSRCU=qs8PC2{iWd`-zrwGq+S>SJx4e?czUqChu}9Za zTpnF4(08BDm2bC5Ko3Du_A6Jez(AFQx(LBHPTN?ea#Q6|pVQ8=o2A23jf$p{?i*b- zD5hDNZ?k;EkNbE$cV~q6 znH8rtOVq_(RrjXX_f1&(abajrt+mnIZ{fc*S5lR?xyl8teB(AjM{$zteM!k-@p@Cf zHs#ndhUOdE4{5Reb6WuaRsKLkCVrduTw8K4m~k1!qFS-e$WK;s`MzD7@+phQOJ#ew zAEw!Dv`jfk$USqfh^KDkxo$XHdm&#mOCkPZ*x1wpjdxSab5YGyVU#g zC;q>`!BIg@PCxschw*mVh8$1VZ-4Od;@#_P!PE$y<}N((R`okSJrWd{=qM`_S1_^` zjZIRXMsF%vH@{hNrGC6f4wWBr-G6h>O5)I9U|ZzEreR6`D=JkdEPlQF^G``pcec%S z_i(@3{|LWw`NF8XHl7%E6fT@UO#6X z5_4EKzDT?|{?XRCOqV3Pat3583#5JIHg~i)W9!QOu>7OBE7g4M&di?1;DEAw_ky+4 z|Hse2o&EXrz~?uv<5%7&+;Ndo2@9LDJj<%G?we#e^ z_lNonX;8%86Zadq?bq>Lyy=#lqw+|5)#rB_Qj&UAl){jyMFu%N8_0K=eUXQ3=)W^j zb>J-5@vzuRp2LpQ!cFp&?mphaX~gyS?IT8rgBES4@iSpg#n$wzr(Lw8 zGqbwm>^>eij6Pm8a97vOXRb0+75@GP2bF#Hf@B`yg5;jAulYFoP({naD2lk3ah!TQ z&5||s@6S4<&R&kf8THs7od%aQjQuM9G#%R!6~i-R{Lj-K4WnO7uL_-a{?F@KNUoSnWxdooW_!hRJ8=@oYtIaTU(|Kt5gJ;kSS{C(V+HHq~d>VX4PKiU1(O68?eyBmk9 zn{%DE%`f&HvK;pq$B9n^1^H3nPfHNjAa(j4=_CFU+Fh8c#ae0ulb@FZZi7n$m!lAE?NY!{0TKS1M?bQUPPi;a{Mh8xPDKX4O`c@2gD5UIhn$LCuI==-G z+cwx@ZZL0lESc@e)~eq!ktbEw;fVt4;(M$EsVa|5KUFE!#PU+M;@B~@2`z38E0c9H zb3GoW7Mvh^5u4*xVwi~R#cPVT5eCiXR{iV#Cg>E1wIX^z4MUp1^TY>#39GEAkgW+p z-U6%#J_7WhD!hO$+k;RHP&k{k1_@hJVMQb8w24LJ%II!N7*O+5J`HtsHwp@zt~oIX z!(ebJ(cAxl91J}P2=x{LrznBIVz+sVXoe-A4FjZ_K})7r-7i1e1e<3h9MMm!h-m8W z-K`Eu=$}V4o=1G=xE3%I^p=07ibL^?^Kn9kp0&{c6XOi%x_EoYaJO#VN`Z#;IdIai zK&c0;efN?G`gwPzHB-7FXFknAx2%%R?Tc*OA~y!pG}3-{yZ+!l*IJp8-_&obs2>wUWu0aZWxx$8%J`*(1bC@#RPOvPH@tpFm*E%B>> zA`j{Z5W$escG2R(xt{8CXvFwM$I7 zBs@NB>f=7<*tw_?VO!;S^U+fG+_0g@-7(IQYkbzd?26j^*EN)%y}$iX(ulEMP^qaZ|wu-6>ewb6dQHYNCIzQeICVAMC18P2oC3gl{un}m8gbkgh zU&26nPU?rgq6(41yC$TK;Jvt|EU729ddRIZZs3Fd!8@a?LZ@t$()*sFmKB~}?ZDgO z>Y$f$EV|}P}#MTd8=QzEDhENPqEKmCa%J8|_hKQXL-Rm{nb%W=3#K_#A$*j8ywmqE~w*)rJQVI5gn=3mOSdK^v zJifSV`LBt(7nEHeZMUSQs@AmMFd6-If2R?xlIgs;TUev-#nGu>f-eGpXYWX#96m7V zFrY_E7bxB>p*Luvvcy>P!^!Ls*4)YLO`ARlNjkIbWNJuN+OuEaP^IikHLCt6=P`%j zTOr5v;(50X35AsGHl1(2(6(gfPTTSmp-O}kNfQaY3F_-;`{w3Gbza%22jBfirsEvU z;mSDcs@!%7z0Jbsymb}6`4Iz_4a6y0_Vb~SF+I(^E=-(whg`jmgq^O7ZJ4z9CfpOQ$?ifRlux#gkF=LWG= zdcz;97r_>7PU_}}!AG6i%mYt86pnB`vH9~=>*MiLAD;!sb^Y^s)t$d(z5iqe|0|Dc z=g*!cN_2_~ZUS63H*#~o)Zt_32G^Q2?;ZyS*TgAr53Wq)vG4O~a$HCuG$LU2#AusX z+@&jO{^;%X235Wlt%*@Y5(IPu*?EDO`=ZJW>8?1Ad(x*b zsLE7N+pzgLgz#M#KX7#WG1JKU%YDlhIvGS~K6ral^v*xc=Irx_j`Fv&=g+sI9Tx~% z0<-|jiKCn_Xx9<-eTau}?IqT(Y4G&dcjx2}S3%Da#x+ou(~AR=7YsqVyntw8&_UUD zzkDn56~hX~XmONpq$LEQ0v12W9y&iqzo-G{$Jxqc0Y7ch8Km4rEC!$8FsKJ04e(?% z35;!OBa{bXYskjtmwC1uQhEPfnh`Y>%;T$zbmfl;_jB+q+tBCs^6l$2?DiV!!o!lW zF{B3|!M5VL#OmU&t-W_xjrkVzib={{V)6I+$4_XQciZ+;LYDV8&8+k99b&99 z&2B1wRZV#)q#~(x#ANZyR^P8?=8*#L>t8d7o9V1!&D4t*`~5+o-e&3;*Pc1Xz*^;g z7Ra|M95&!2J9y~_4%X>BwIH0EpgA1@UldEc#CJE=^r6$Ix57NF8i(MKs#drYrdvPE zL`(_0WPK+*e4F$1*!j`1G46~RIOY)dMZ$4MixifN(-1D#4X0^rxHfD@C@l? zTWlk|8$z)G;j(019d7K_fH&lu!A2;NT}cjXHl{>Iv^xn~W`4>FBG&H#m6n%N@QfvF z*wbgugr^$@@kPhPa0ml%q=yzEbp)itt*-u-V8S`54556k^XoL`$qLt zNi(@0TL4Wa;SOPJ%r=LSUN1s5hW6libFO6K#mU0csH;47XzVP$`DOLDQ?Hi~8qt=o z7rwIN=lU%HH@ESOY~82jp`*Ga0@z#B`HrvkUQ&Ts9lI3hC0@D|xn{P~blLcO%brE+kMf*tpB#}A-p566tIO(^dOnn+Om4ntC^3a; zCre{KF(CZ^(f%{HS#3A7zCyqzw8NNuQN-x!l_N{oJe2tvdobmBq}o z(T$l@>^0e^_2IsGu{GWJ8?zJLr=5Nu)D^}kz<^>y@{)NnhS$)=bBm z@36WcsQyK6zu1{D{z++>%GWR3q^iysm?cQZ{21w<*$`R8#vFe?TaIdyc_g|}`beAl z$sK3+_RoY}JZX40tf%+Qdg&_JT5;*WiCAhdoSD$wHEPKhx^Tgx&|nEu=(eM-gc*u| zrN}zxrGDoKc*9hlmj7x)JmE$d^J)coDw_^}lm5DJ&~%*c{hdxaC$WB?`@d$sJlo5; zZ}|t6Z_|GL+?|w`dchj{G$!f?cVI>js&pFCICjb1dGJwX-?M%JVO=X-unV3DSSp#^ zZ$I~~{m%`_JArdg;C2fqq+#fgQ`y1k^(aKbqnd82nk_!pU;KG+{QCydU!2$nc_GCm zFE1}5IVnYfjshb#!kC6*3=^qXaO?-zNSm}>%u~OMK*sISkVrLBJc2cJPyOkosEvgV z{Mb3AU)zVCn9vUV>|B;LR$*XzW-KB9Cgxpkb@GIB3 z=WkEN%k$mm;Ng~ymseE0p1bAU_5K&s%&FSOeTQyPr|5ecLwa^yc{0DP23zjQM#H0Z z5#@m$?<(qJ{=(4(2)@P0UQUQGdJ*m+ahwK?@b^IUz&l=o*8ue+YCp;$@XkF8nC0yW zv*zad16#iT@RFjq1pW}9JRgcp-QDkV;EVs0jNQf8>3S%Xy3CO0dRd`i5w|uyzOXCp zSMrW!lq=fBZ0wD{l|*NptGPDj>mP6ycUZQe_g2JVL5;KR(w5W1>+^3by2jq%-DzWz z@T}3`g4VLy!5z_=NjEpRSPMpH;&dPF_;xuu^HKQPpN)eHYwS0xZ(1}nk?8jktxc&* z4vf@5Ato`4x%CurzYS zfSOnU)sW_{K>$9%^L8)ksl9CeZS-rbBU9xQ8YrS9G0Rp<3X_hzS+D(c8RT{sa)^mp2DB?j7)FktcD%Paoy7< z7M=NaG@$w_PdJCVlw<<~6Xkch`3jNcim#$wsULq_+5~b~gmsUMo51DO+DEd_ZB?I`epnyG){KcJD#B;uzCSvE9Pn_;MBF*R;|Oj43CLy0c>_wPeU zL4X076Z4Vr@!SutGfvwOF26Gcy3_7Tw_NCZb;7Mh?M#_J$IBUXpBghmUnG5trBL8n z1?%hMD`RIO>Z_1bC^pJzXx6n=4}CajJ+UNuzv4t`V9k(6%1f?ww>>=<{{E72g2wzi z_dgEaY#UfvIaLd)wwrEl+o1x)2^7wEbURhae)27Er(*+wa@)M!rO%ANy%}&lv%=>6 zlME%@+Q5g~ONwuwJp4t}D`C~T^joF6lE1po-@5(a?_T8sbP0QWDI6aU<+f38koMT6 zvXeW)eK~!V0BOZ>J?&|%T#z|+DJ|YhcE9-b;pgY`((m~G{rv;;&0(Z09kr`4X5t#2 zH`R7FgweZr+>||g?azTE4(;f5qZhccUi_t9E&wxiDGnDYkhigu=9|i&_hiN|nw*(- zx4)!YcEtZa=TEJwV$EvxYCKsZB}11m-o2gTx|}UFd;iv=5N7hsUb0lx^QFhtS6#R1 z?EdGlmeV~Z>T2DRt^1@BE;);wSWUAuq8hEflB@bg`oA`4l3m|=YG-FB^oTpiL>D6O zMSC6Qf%t=K1pe^Mt5#M5vmy*f$Kp9!Dhax?6Mr(-5JfW>@Jqmuh56QV zs2M`93@n3SMlWvEy<%aSFrsMcqKn?|{Vg)nXo#2<76-Hguau|WYs%CM&)atYP5Pa` zPXz3g1vhF}g{8wPXC>I$wI8m)Jdg~NDV>5CjOb>_oDK?-7~-Q$LJ)LUFX6{YSiZ3R za`%=%z#1R!@q`GF;^ODW1hoT^Uy@-m5T`tc_I0bh>BRPkfp<|6n^lh7cOqd+EMux}+Fy55M?@CfVss>HnikA2HBbpcn_%FDp?)q4FZG;jq4;Aj`QGWct6ECy6~p z&@y7%lRO&mlG2VdckbNT4T|!vlTRp=Ct%^Yu5I-*l+k;}Ror056gQ3KqNJyZvIzacTPP@v1c=8o1x^o(pTid0;tJ+gz{md(to3127H)<1n7s1# zHYnR*NtA{=S<$ z-PU%b{~Hgm`mT?Nhes?2D!x(Z(hj2jPSslt2bv56fP-)pv!D<|Q^^Se-2*$Mxa>Rs z?jJTxVOh6#?_LI0SUz7vG3?>ZMeTc~5qF!5c43#?H&jmbI|CMGg>KLeY?-T|7WFh{ z|JV6f&MCg>v1%8vcFd}Jj^zQjrW0PJm;$EJ-v=RN+^+!JUu2RI3n7hoednO^11CKc zVIEh%^NrkLvvVFsGvgs28OtsVE+_c4x|I6ZCc$EBCo`7IV(zGQ1}j*8Z$V@u^8 z=PzJ!7|{$ISOIYKHqx2%kzfm){_gAAI72 z(l=SQoF(*XlJKRR{O7IqJSQO*1fN#-uP^Wl&7(2@e}2y`_@1@% zj0U)f5R?LFf$Zvd8pBulOB|`ZaGl!IQqopr{;tSjNVd&>z?%>pGbbt}#P{rhEo*|O zUWeE0Q@`b&ynC(h{is|%*)!d7n4?zQ_^c=lj44RojJhhGzY`aLmdcF;hQf>~@IL2rxZZqCoI@dX#Qgu3lWLv;^Fx)zcgCKY8g#Yuo|0VA#zOj(f111z-~O zd>RamK#8m3(8~D$G54D?Ysq&9dC>seq}-J!-71dUktD#uXioc8hy-6iRfvvh38=1w z&IE77xZo;G%diqZapDA-X|sN(9BFoEc%b+0?O|`__r$};hcG{|8}GqbswNOt1)hcb z)qDUVMEBdvWM#3#q>uP#Li6?nJ`x$x+Xi3_WqYk-7k{7tN5n5Z*d zgm;_rodD?_gx%wEV8f#(dh(z+Kz4=m#zy!Y)$RdMHrP8Avt}5)t9y@k?Q$-FT!~`d z6-&siOHza4F6`NzbuA?AA%M&vug4Eg)=X#ch`cPjp>CinG4?J7ww#O-S^&no$Y_5pK~1PptLXz@Dz^7@@LR6oBJ zxOfD-YUYeiJ@_vNNMeeUg_YG`6(r`;Vf2y#+G?ZVNs&eeWX9Wg<)^xM5UWeHLG8fl zR1Cv)GHrlNw}btE^*=GVg^82;j><6FUIBq695}sH%9qHaTIALZO>jf%C*21S{8=bm4Zo z;!Gs%Cj8nWdU`s_(`V1f6TdRilSOeMz{;C9Cw4o-+3coB&eUC|l*hxn|5*#c0{)YwhM2tx5g@D_wB zY7GD;I2M3kq>tN5{H4&yf8sD?LMH40Goydk01aJ)~?g1t}?p_$-89HCEtZJRUOn=fdagwd>rY7U@BFT@n?sdtvCq zt%aB^)aA(-v-Vd8C1JC z(bmziM6qh7D;P|9YJ=VD#A^h^$ACGCxyiaB9SvfOe?14&qlh;fnR1G0H!kpILvQIi z>@~h&=|%MaNL{;t0&V;Y&=J42w-aNn?A%=9WMDVc<##%1{~i;S2jYMfKnozYs$i#+ zBpuHh2tN}w%dR5v6Dk`z3b;f^Aln3+bOqjn7Oz4Nk#Y1#rtP_8sHj}vZaT_r^&n!a z&@J>a^Wt)h^+B#{ANLd?mkhlmW+=$m7&bg%Aaf~Conm1*V77vbi!1n@4w<$}o-$=C z+H0__V(bZk>nJdCf*wJ;KjCP}9<+=pF4fI(vA#@um3c_bE3pwkD^pnr&(>Y zPvA)FP3ardm*$ekIK`eP-n<^*Jx`(ORG9`5qyk(F#NuC=y+C1fe}JSJ$K0%c2WGC^ zNrQtBd=DlV3E;BaA0 zaC0U4(~M6SF(4}v9eh%|AY%g#LZsu9sBQ@Jd24*c#XIzxg;#{IR@2~6jUvDJ)vHC2 zVi2D+GVE3W_42)Y^jCAN;CMs?CBNYcfoYTrk!9kOu-AoV;ilSU`XJ@61c4TKnKRD{ zjm85bUhx0sbr40>h;4Zj0}+tYr~bKlxWz`VPk*XAe`K-J)HDfPw=tWM*u;aN^?+s+ zT`?!IQ^2@lrt~Ax$2pe-Ze-cIucIjXj=S-ur9LOF=M{{fI>yBNS-IhxfvDG*6oEwe zvelCkC4bDGddFxXI@1+-w09F1zG#bnI*@g;s%G%2kCk`_(ny#X-h@z=yGwv9It0vO zgg9d9^&)eGgk_Ll^!;QZqR_XNbcXXR*2g}0j?f)AkhP#_s}ttN0f$}$kPw4*SHvg; zGoT3j+1ln;@^lbxD}LSU<)s?o>0tQLkh>u zX_)SwwiVS)G2r}VU*#*k!B8!(t6PubkzgofNH0#^NObu|VfG~H`LL|Cl+dX+?!T~+ zl%|-$2l251AcH_qP>A-I^MwCtoNBi`lvN3323G6DK_3f-(ZoA{WDkGNa3(ppF*6ZF zfpy82I*d3@aqiM9je@z)JtMetc>1xJBQwMzq~#9%{@ZnTlB_JDgZ4B1E=aq1;{2bD zKfA*f48N^=^N58xX2~nhx+UVD=pSmw^s~-AgJXlC`7mDqi%B`FCm@Be4RF&4a)fo@ zgWE*)T?NWWDdEuIVlY)kAD|JnBVHqQ3%v8=M%9a;*+4tijmV!Vh=+v>y44xqqYXw! zK|)c6XUSB)6K+Z5d5GBjqEIk$Xw#C_9;xSK6&1@+wi2^F9*^Kh@QQeOb6Ma)OAgBE z#uVRpF0HKINBpU-Zf>Si&cqH29b<~iPWhmv-t3^F5=A*$AdG#F_;f%bz{v|lh+5m_ zjTXks@^YEBNZ<^os>a5LOiMB_1%O9-Comln1Cb}AWg>>soq_GD<}6FkbZ4gDqKld` z_wKf}$FZu`3sc!=f&Q`0tjmnQ*)cn5whGO6wSIabka2hU5S~LP-e>BxiHQdkPTvRG zzE92cBCmW5!%O1fFwUkw;46A{YG?yIxFsj9{HZ(;p*7W!OoPSdA+z$CUwNQ z=_5-Twcp9bMOa&q`W53rq%@t93qTi>+ST;{JR^AYCtficDEuKKD@#<=#6J1hvHY=1 zKqBtbH+F&G@_TaZGHm{~z$wr29yAoQ9A(q-()Qg?p4<`;{KCFEXT=8@y%XauR#A3i zJ{G#`YQs3_*|1Mye$MO%O$*{0`N5OZqeyJeoI9s}^R_jiB~wk4P(`1{-+)Hvs!V9; zIROG%dIh;rM52b(VU1;e)zwNh$-e~Dlym}7|1yGhqi&LiB#2~kG3kf+_QpTdb=^yF1MYnn+!MO@cjh!$ToWB!( zQrNL5tru_o+{;siaT`y-{2>_^F;hr5l(&DNFpd9yIir?je58q=E37b#@~*wOnq_+V z60I*qc_S9}Y=@ZOm%jou?ue$&e4>@CVlK1USJ>>HJvG7|M-fy)+)Q>^N?aLT13y&Q1}zjH{~4(-S& zXU?n>Nlg~_o(cB z%6QTE;Rm~QOiTDTIKn@!v0%vx{zqH64ye}n2W@g&k~)2aEzA33(b9Rcgoa0?g(FOa z)2RpbqN|P^bgI|uu>4tzRv#HRN_md(y9ewLKPOdH)ifj&Xq-Gfsb8*TuMba7_-hxx zdzW(Z-O->t>uzQWCg|#LKfJGpqJ&9CNS=7LQ%ExpLP|37nyo5t6}BN$87fA}EG1P= zb8A`9E7Ve~sN6U|SVnY^pu$|bI$o7)*6IVIs3}{1UNJG^J%`nYB+LEXcixM$&Ge8a z*$3#QQzyDRWt19qvU@b1Fn@A>$fEh8EIAR)0L^kn!?|{L#E7R{kXJy*dt<@&wjArb zGS57@IhtSd9M{vH%781vqOX++x6{+p^+gmd%Rf2n;^R~3EFJkYuBUqL*;@#+fz ztM|1+{iLT0KiG9KP2Xbj-MNQ33TP;WOs>E|7x!xfL>D>=%l-=|<1nxkp-VR{ zxe2ojP8`_Ci?G!S4hflJkD}{@2y5-Sx%KC{gT`b0B(Ffs``d54-dJ1t7jOC&IxMY9L2R>d4D3Gef8+F%2x|RN6M=651KwtnY zMr&qUnb}D!eU-?+8J&=wZNp?QNQi2eXK$R9Erk4J@H?`D)_Y58yA*ZOXQkSTm2yg7 z3bMk2KS35xUibK-|K%0ej-S!*_IU7QbM(}@V{M$rG5Ky4&fknFI0~Ub4#H}b3OD6B zT!-M>w17hRLp9UHm5xFovtZB`NDw6>f5EvU7Jm^q_78$7yLaDAV(2&$(+i(6R}yOw z>@Wd4zo0sS((6zYZKM!h4vu}Pv|Fkuqe$;rIzr&$M_S^J!!l&ZGFsu)D6B}SMS%tA z4V}D-_fGy#bDK95yoeYiapl9oh3mu)@u{Y_MU`$pz{56PdM?7?GgbW%xcRSQr?bQ~ z&UrJg)+>1~6;dIEqJh$#NSFNe{NXload6=8Tb)3AOjG5pgd1TASsgJ!H+p8J5XxT_k

P zR_{1t;DZ;Ob`N~ExB*kl9Wu7f)&ae_-Fz}`o-9}_?mHDjlE9oZ_d?zt;r2wEw?nsLx2L0)E!Y(_Zq9A|rw?Z^(LxN~zm^|&h z7@&PgMs02K)2Ax%hg2(blut5GdQ^>7bw})M#e~ zR%1DnGW{oX#t7Mlu4ZaZWY~hQH_tzt;Mq!TvGTv5*}LV|E8G5qrMp)Q$zOwP5S+Rw6MoSJHBvgdQ&_f8)(C#|v$nEl&qloy?oMLy z>cngsWDVh{AAdI6eo3arO?K^cSTX3(n{n^aBdsB8C8fLIku$Ngr(zi6 zPoxWt@V;SUW4n09)p{S8;(q@ACScTiczF#~JVu9Av{y`O!?==!YhsGdo$3plT(`yd zu6#n_xiH?RR`Dh0CGY#G@dHeCmuVRWAyb#`=}tSuYyC=CJ2hCFkukCIx}x*cBWAN{ ziIEH~pBkNxNu@;l>MLwxp^?b%JonL=cy}<%Gga{gQA2bD)I{@)JnYPrHIVmYko#3I zSbHM3BgFAa;k5hcukrC`CJu2FBW6*-82&fnHtc@RTgKy(l1u;0y<3<+ z4Fw}DrqVFw@3ihu8c=8V+g~Hrz!&4gU1-vnbgGJe?gyWb{6p^3WU*x^&=oZK|lZ|NyK_w$q_sl)gtr<*Pec-}cs^2Yn}FOASu z?c0l-zXT*-_kDV*WA|?3>)e_A=81bRy>I+`dntvI`D<_|U97&7>?!p!WO7B-|7N5>p2 zbDsYVuk~}Ea9!3|VcfEeetaTcD$hS=^b|k)Lg-dwYfX%UOAYEY*^#;(L06 zxW8(RfrY*FekuLinbxxQF<(B&lWsP$!8&C9{I@L*3tJGT*`Iwhda;zVOv9jZywu^6 zbI(jqf45V;Qlj8JwP5ZCSvJpIwS4EBKhJ(7?IQP3g!Vr6Tt(ZZ2BA>0Yd(hLDA|`h z{P(?GM#wWuYMNZw#@_N$?B}KUT_d9+^0^tt;a@2dStreu-EG(p(mJjPm#$Rt{H@U2 z`8rySJGB1auhKHRIo31Qy<<$2VP}%aljq`m_2~OhXQci{*TtOj0wqGj%jYi7NSi!o zn*|0RgY=^XFUa-vh*Nz#42o>nwd1G8iW*II=WkETQL=N^Gj^YGlU=#FPNPvOTKtsM zkWyZtI6{WbzHUMO`Je0y$DR4G?2fUyhhKYCqtOK{>X_kf+w&FQtlsm)9+qj>q4L!GZZ%Emd)Rc})?6mIzL4Ah?($fRyR?^Mt}A2GhN zk*y$dEhMP6@%L-~N0q;`PZ;W$OpV9Ay2%-LIWatD)I&!gc77YMEF@1M`1xzQ@^iMH zYRv4uk?Bv4w@%nmPK-u z4dE_7RXdC2Lq?AN+=addMe|?%4SCeN?7JdW>e#3~`rDZ{Z28X2lW1l1DP(HsT{q)u z1U`j_yl&4T{`&-x*-r>-%I#Cyv*M_eaFA_$($EohKXEob-y*Tsyy`UnamHnFhyZEIfG*M6vT>CTRw*CKl>oK)j0ouKEe|xPbrN7UJ*bAJ%!w9 zuZ|Dy7+a?Eiav-_>5PfSf4<;HUMkrW#)Gi)^M^*=O$7beHN=cPu?~v}?!f-#T>clwoUK94~3dH8R##o1hbFXXrft zIfCbn4;zjRwE0F|ekWAMD~?XL5bG-9*^P9H27w)4-mkqv!3t9&MKc*3Y80-m2w^6d ze+29p`cog+a>9v5atDR>_CX%o%EA9u2f2K-B@MGaHKYVjxCJ5q_B*rD0p%VO3kwyt zLt=m4?(>a8d-mi#s>}Mjv0JEZIS(%hu?dDmZkvdc4)K}#=wuQ7bil^$;>!7JEyJ~P zf<%B?Z10Lls=AU=7#K~rTwP7zccIn~PfhNB{*m6aDPy9?k`m{wrxZAOB;vC^-}&f@ zRXvIQtW%=Qw0v^u8Qxvt!+V_M`_+D_%rDC;6}8u8k!}tQ3}EhGf<7v3bO3APV?(Hq zD#Z$r&FaN2{kmww#=7^6iSh>HEo|QnMH2qmC1bo~b?0ek<$}v1FYD6ucx);y9Qzi{ zU2yf2UA%Ety`MD|VpY))Pw{X=Ygw=Z+mG?{d4W_rns-_z=!v-y8Y#y{8jR|K+;+oQB=1SK8if z`rq%HLl!NeFGNLQFs=+KanzKR!(kGnjjGiqZ{+ICL}*zs4@HUnaM}=m+~CEprGvg! z9wZ}*e(PJ;HrVwyQ4@WZ9=48&DH=_D(}5r5gzmX`^_HaA5C>Vpp8Vw0*4F0GS1^LF z0d(ab&!_Yj91OE($GOIou-eo}x9hSABi@s~QPXVv_bY#2_H1J|G&=u&Z)(clJF%~* z^-Jv&fMSUdcW(I;0zW5od#j{2(HSX-_9$^z%A_{1ihas@P}Cg5P{y7o`E(T%b>9KK z}HC;28uhF)l3Zc|a${We4}}m}#$0v0-JI zelvEztuq!wGTGIPdO4;s|NFwonnBCh7`jJ?Uuk9aS>}3pB*qX4!T{CxRJu_(KB3F& zKudLq7&es!B`$I6YI?ugQ_QsS9hv(}{-!egii;f%4x3K(rqa}ve>7On-@xoY#)sQ( zqaf}MciU2nZU@S>7<6uQ3`*~Swp$Bb!%!mz2!IvHB4Bhk#G#8DJ&$q^0Xl*8EluII z$@0tTmyG6=T=JBC;IK7IH)l$7{#3y+xvSQ7&^$kP)%<7K`ZLnJ(N`8}w)u21NaAYY zD%OKxoA#YE`kZaauzGbPK79zr>4|`jVE%T$w_rq<)Sn8mbzSLnj}M8$p}m<;2%~B zP{G3@=8z7mQFM1oo&vgH8N!AXE|=#`!!ai>+Qvz%Ncebl7JQPD^j&O!@bAf$U&uW7 z87{>vYKPm5KsXgfd>r~RGpzw!;#AJ1=xIzP#he9QQr4jZG*o$j)pB9jv}mtVxkBoY zsFScmQ@C$eGs#Z`oo{m7Xl}fOymjZ$<e#K*V9)sAFr4`@QlmlS_N)lMhTAFY>uQIEPX-^_Bk~kEN@uJedO2evZ8T z<-I8U`Sm)SU7`j--2pwdaSU-Oeflf z{l%?mdZq=R19g3cU#+Bh-hOAcCqVmD@PK<`<9yczte61u;7Ih|eT0h>7K21Xi7*q#ECMq*4mH;=j0jimY1Q{cQmGi@4@F49hV z4>*#%8Ftvi-W(wBLtTlf6;)M=WSYqIKtR!KpbpNmiL*123w`V#Js>L;JRt1oK!a0CVDUwE3p+J>Ekx%AePq0(y)wa76W3hOdj-k-KZ!98X@p-t~-~ z=y2jTmbw$Q&yW3aG*EBZAALUc;@zS7wcTDZaNT2vvESeJ zn57}_H2mcgBg#21VEpB>djx%=NHVGTRJKJp;$9*imbCBp)>c8dxi{OjdLar8T3&u#QBjL$mP$4h z-hlMlFg?VoQn2wS5GRS=s*7MVI6yi;=8a>SQ^B1C2e488Ky%ay{(|&&i4|cF|De)c z<<+Tm+%+DnX!yqujkzqnr;c0aV+vb>!0;#etghhpmn7!H;^RSZOu4-wVGwPyaZ$t$xb*efX2 zT0sAO373$Gkr4y54AJF(C_eFPpA%9ahNVk?9|Y~As1f7eXxiGlWIS%)R>EXcGC%-a z$nM%)=lV~abZwZzeK~qR!T4u_&cA2M zt9r+#=J&&)%CRIjkGjsmk8eH^Lkl=THe*t8kkI8A+-e_@(r*^PV2y!yj{~+y$9OCz zCZ>469_8ibxj!R3cgP^U(ap%yMp|iwy9{d~EqKs3!pKt%*=s}uN5S-XH=#T19)afBrNg>q}zt_l=ao zagz+=%h}5^D$l86UcwM;;7u@r$u|mJPBnCWV}x|C^h+YeQiN<}95-Ij;_D$96%-_9 zMN>9n!%YUhQDoDBrE$RPDsm@`6^H`n>;&#%7j)85#d+FW9DnU;wyKl&jUt&KNRZZ} zos`NGr;}`L<0e1kF5VTlXv{5M@#U@A=ECL+@^fn`Ovrec-mlC1yKo4^vO|4(PLU?q zIQ9c)WmKRi#jqg#-1CTezEK#J_RpR@HC9vJBE3o=v9PXk6Q3S6$TEdb$2h(^r%YCh zLn5MtT@AqZIkf*3gTYJd=aPZED z`rUPfDw>kYHg=w_+qKc*L*o3^hZ)0((IXTtJ zaH)xu-7^K;lF~T((!bA0{ zL#Tb#1RW&CQJ@Z19{BRJURALHgdSt-TtR=VZXB$tz5;0XgPEJH^CfS@XSEuzN39-F zsT45t5}!P&77!N}K7|#wrEH7X^v@v*jlUYjm3$P^?BMTcIR6m4tVZ*%3+tZ$&RoJ0 zJR1~cWNMf+w8Ni1-HVD@8Fk7R>-LA|k>M7z8=X6Q*677`dWUbfHYD`QD<}*%nMdv6 z=H{-MiyKMk@QY#iP>iFuc*QQTt3T#=7m@-KVJEA zmK&gQwzjmc*w0?)mv=tzm$hBXdV?eKX@VmASO&i+r0Xf(T2hi2Hs}*s!upPZ_ru_| z#n#ygFWyTQCzL;1F0fxT%P7IlriRx4DUY7QTGxtu>PDM>Ci z*d}QIg{Ckqvm7$cl%Pit?KA*AZ@%<~_v4c@HxxvA1pNH`I->)0pYCmIO-M?rkCM5c z4$>_6Zrz0W@o0bh7nLQ|2gjs>W(w3f6X6$KwsK{|BlE&|z%c?{?Oi}DG_);mk#igf zmoZ#i04zNdZP~nKi|&xMrDa;D^pnU`dzQ;Tx+7EBO($y%e9GD`vHcmp$F>V|3j{;f z8_9VO{Tk=AYMhoR(Ux4&HJNW~x;~cIaaDFn?VmUG83q%=l^W}}E9C0Fu`TFiK2fR5 z=Vp~0Cd9iU^QnK8m%U}}iS*U)17;}noEt(wl)qnKpu_kcF@ zsaLA>*@?w*51*gx8&bL_I-+d+8ny3(H6n+2IfU~{v@aQ3S8%P&oddUa;;yM$ryUvduxP1V)*7y{xeW4%oFoKpePDw95uOUWQV4Y7Pd zwr|kB@avQ3aqoSY@y`+)Sz#1qIc}|it&2a*(6Za_ibgT#;?aU|Lujn$l-`7 z5S(ma@@)Iru>)4mP)Atzejdvn8TT#zR*f}63Pu`}DkyappDmF%sU10TeTOj;vn_qx zRBA0K51i#zR!vX~z{Bp!+kWtR-vba5^7X+D6*)K#Nx!1i>wvn`@4j+ensuH0f%4rebPzZSPy9ZJ zdol)y458m6l%Xsb9;4mEEcdiyriYWPh8llt9e1b2(UHj3Teohh8F&xMjbTt)0!uY}Utp>P5|Rlj&Gm*q_3@tR3;_2bt&hyKZs#xlN{)24}UmM@|)Uk%{!1Os1+CjW!^QLqyf@B z*oWp+s%2|weD*y-SO5XPlqZzXKKSQlgjr7$OHBKzo$Sdg;zIuDwy7MP_mN}A}#v8eoT`HgY z(pAK#oqRKUdS&kZ{KS|lzUsi=pXXvMi4PvrIPQ&FYu~-=H97rhblCnvYR*{sv1!;o!7|!hegH@nc zK6v~%iDa2NxmGX(y9IubAS^Qq{=5?j+IL_OS=FWeIq*{9En}Q&U!G%>WzqsybS7P(H}0 z$jRNAoSNc&xAbc1DkcFn!Q_`6eW#-3RXTTsH%PK_Ql3i^44PG zPo^V+b}ufMzt0jd7~}0VbxUukYppxGZ&zx3q^rWKFI!)@`}{v~?iyO~?JBcppTY;u zqy8mbOSCv7_t3j=E-?7?NZ(LsleM!!@5h*1M;3hkQniZ7k2-Y7EVw<>MI(qwl_pbF_)|x+@8R@F(Q|IjcOFJYkPce|$857#nWTUb`zL>Z!z{rzGLF%KHl8Ett;AX2<*Zv$6ckK z^i4e>uJ7v1VWEH3>z?CiSGKCUdYy@-GV2Mm*yBbGucu^YIzlS)S;u{$&bGJv{Neza ztACpB+~gLOen^yu{~jufiH(oH3u-wHNW$$%L7AOAZuE^PFlV-eMs?HqdrXd@VuDeXY+dX55- zIS+Iii*mGw+4HQ!YvsX+a3sq@4}okTI_wa*0uew$^P0i+%$3*dK|Fg?(E;_7*k4d} zT1hdx`}C@mRa`=tO5gB|GB;LPajJXG>cww&(~5)&(dznI>15^xmWoOLl2wQte5uXt z8{1z2I9FTcK3zH26seD+JaZ-n5;4r6Fc0Oa?bAJnDr@K!gkC+$TLf*1ZI`TSVu*${ za-&M)@e$Cbco{BIXzA-^JAnu__-zT$KoowDGJt9Wx&`eL`;^S0VG@Hx(&%%`?e8ks zP5pIOC!K+BV+>qPsbJl2=*GS5?W%RDf=ss94_of&l$<=)9VqiNpuOs=qeOkYjQ?@J z`}ybohn$ix>PX3fM1^`^&Rh$fW#8`H*fZH`5pqeq@9E8%js*6LOTa~%gLp3WXT)E{?&!$YN>nSA0cy+cBQ z73N7K7>hAqEWm5uvj>i66k&AMf88c)Z1izd$4~oF4J@Ejkz2{>tA<7{&?&LhmdO}1 z|6<5%?jpZ+2!81aO4Win{^;QxAD0H5^Uno=GBX(9iyAw)1QZW1^z_lRWZB{g z5_gk(s_vB>LI7!9W%*|qyX8)0@5n6%3Yu>Ix{eg!;7}dEzcRP<37nV<$jvF!UtY^D z^7o0i@ZSMdkx0YgyJv<5Uhg*V?6FTv6t+VQ_Qk3WEV=OiF2mxeAqspZHDdhrvF}{d zU`*aD@BjO6eZ1}@u$IOySg!xnWjZdW(fjojGpj&egg_0&%_@!gcz)r-M|cWI$6F_n z=UwRUlXMyHFki3a{=~d|JR?Izh<+=OTSh*qGRgVyZso)RSd-bca_&MJm=erdN8adr zUJr%m6jKbun7NtmOJzKiU0h9B6_idd0G3jfud<0FNTWjuI}7ih3DFn0zL=cG8n}k3(Z5Nle$W8<%-Pg2F1+j( zd8iO*4%wjJ3#uLpwaiS}+%(wR46duMUj>nucE}me@e9{d^B?8PNb*`_vDXF0rQcus z*euZGs(gp7`?R=xa2h%&G9u~uhJ=rA@>W?s)hJ`4$SE3gfQM`rsRT_Y;X`Sy`ZHWT z+CMjBv5kG?LUY!unLSAFsb1+!j`UDjSesB6oUb&#IXN-6AdO?dTY1BJxA{PKK&^Pj z@xF(HkQxlG;3m|GIO+LUxFJDM7iI)*&MTiHJFQq)z#;pD%^?Mrh^6>u7dnH*>dmT% z!G>eG0O2&$sI9egT=I!{=}-=t{0Q z5i0#pkX5wbOkfw(i^(YMupDsz*E)~c2(M$MXOY-A*_NQc+^ODFV9LgHLAz?KSS@?0 z?xFSlE$t-x(9w?kARAXnQof4lPZ3!yf#L%}QNkbRN-DL6p6H6z9>cCIzorejrSHtV zm1i{gFzZO_K&KGU4jg3Z0m1CT^WY;Y;z{*6zyxhR zenUmOh-Hs^)kQg`w@Zjp0QH|Z8(2SvGd#N*iRU|cMXtTo+P1U`cVj%|1CNfOShl{F zQGFy4nngIerF)hP>j-;+saq; zRNw1#-QC}nWAF4#D=tQ+hP0`r@TNqWI}yUlbE@Z`voTBUu3Zpb#k==}mDUP-uMd@& zx0{ks?8X(!ItVRd9S*=qj>ua;gJ#2yhYL{un^4 zU>bppztGbC6Q-w7^Ik%f01T};73De{-=0OP>(1S&kSt8fdl9bW?^X_EvZ2w8eC4Cm z2+!6w6ItIY!yR6jqK3#^3=VwP@7}$mEr)s&`krK%P3q4CL+U2GoR+iUp^nSzp8)tiB4s>jq&Nt>@^#r%!z>Vi8Vwt4o zj5b+0%i?`+Z?R9SJBsmt#m-j}^1g;*w;G1UcM1gU_V4}dq z1G=UY_B%*&uD!BwOlj9J02*>N<-LCQjeAP8GLWs*?^c{?MtbU6uox*29#4{>U0$@=4Xyf0ClSHi1r2 zlPw?5^aYS`x)kGIynliY#$@>TPwQtDmlmG8EdG+4!aAsJ>|Mj-tSLrwWk#CFPg3<&sikXqaDJ4 zsr)iiNIzdSjA3=7a6F76ludgKt(JuL{kIM|!9d?_cLT#mfKUY<`)SwW_k@ zVP7rfsG-+eW?I3w@IL8{C(EkEP4!R5-j-NQs`@XNt?b+%3!ZJOIuWH_|4-v;-O=um za^>l$26Q%nV?#w}hyk5bOtj`xv4n4=L{zoQ*#_!J?Q%0|pJVpB80OSN3C&BFDcgKY z+SA773AXtz9Z|i+u8AnN$`3PjF)CDae`qZn5yiHaa8g3nUn(f2VSG+5h}Rq5)X<~5 z2Urm@3@^gl#`d(U>SO#O_oJe!wTXpmbJ{7slg$pBT`$sjb7~CoMLoRTd$t*^n{?v` z=D3=7Cic>ysLUYH9<1Wq;%0wJx+5SusFwo_&mt&Qx;*ji8^Jx-@aishek#3qGzj-q9Dw?622A+|7b^454z6Bjtm^WtWIGH zq-Y>mcqt>B6Q)lT61ZL_>R8! z_M%uF8)t?O&_`xl1J9>hGL2y+rHM)XB}Wtp;=ajNF|M!bO$&!;ikXj_wxX1*Yk!4cyV1vi;Xqlzd{jzpBhw`koNKP6b$flzFGV&ii_0e&| zS|l111-jF7b)*~Yp+R7VJ}h36b!L=;Mi1pHnzc!$wMDv_+IXg^^xVeQrLlUunoMJX^Z86<${i01)O}8 z^z{9nTE}iT{8p8xqI53@d8@>y?WM~n<`LDX04XWDb;t-0koE{QQqxpNt%YvSJ)<^0 z|MiWZNNIVl6C`8ty5mpMwD>!~;3VLk<8cf;%tXctPX_K1+ zm3$`5Bl`Fy?a*Q4V%IWm4@TIltzF5Ji_$-{Egzk_f^h1B&g63CwUw`z{U^bPtpO=o zlyr3Ika9E$H`rxe*nb5UIHWJFBfm6zUqOK^Ig2xAQXR72+FVbnPk3#YrWah=jl%WtN#gt|C^bMO98+$pI5I|oXTOC z9xi@++sz=62*pc1856A>8Zn>0vE{(SxFrF-(*DnfNO;jf&voqBu?~F#fe_3Dayf&9 z-%$C(z=NCnIl4g4b-pc2CIPyi{U6PS{iQa2-01w;*Q@|U{MS>=CFxw1@Jo1gdv0UH zBGO=)Kx|@}8_#~qp z-1HS|o0_^neW5Xjsl4Xr>~DA$x;4!!7jgnmm*RE(;E*zwibTthjMC7}2l8$@SQ(*{ z(`CA4e%;iYCjV_{$yVKud8(Zb(|e{gZd6?6JgcEHABWQ!Esr|tH*b{2SJ5Knz3Mqg(_6J9c5NC`IXhH`F7 zK`A(qGd$WuFYn%`V`s>4=MNt51EMH`;xW%e&Sr6=&DL4_tOmwEC$nsCH88|^C$ZNEb5h^r%8T$8~(Su)yB3&=@hBOI` zSR@Hrzc1zzy^ZXisig|70b?Nkw43@tpId4&89XTZ;lkyEzZ7lAlS+?9?5)tH<#$0y z7ggKn|m>rS)E9XGO&z@igknOGDLMnJU*BccS&~jx;Qr zO4;=_CH{xoKf2n{Ibp07PW6--ayzYFua0hF5#(ktl;@4%*%B~kSqeLV=nz)*CTuR} zFWf*S(}RPdY#^i})Cf*HM^kg*=LECjGSTV;h%X(a}HnGGUSW_z*h$MlAK<9 zCL5Anpb^q8w245@*dEJCdbBbx$Vd#0HskXB=h9?|%guaff*D&&eXrCGrRQB>JGi9i zUD!Lj3vHu7txI+0%+O*9ROt@0Z=cjyXj7mn{yf}cI(T7m?5pV*mdkUlpY0xT*oO!Q zi}y^ig=^PIxeu>vx(TieNCP4wA~jcGV3VPr@6a#oovSG$M^(TE4oGfAQI_ zO?KI^a>AoACk(yqjnm>96s`&_F>`WVmz7uVhzK;e1S@!Ep2`M1|o{=bvGYFWK(^7UHZ z-i;BQ%mPQnoekz$zJmhdcH3~9mox+?jM@v#v=bXcWe0$sMO$^B3JT@c0DGJG)~KCR zq2xa`)hn;d{Ois{jCfCbxsPA2Sdv+rdY;e);vDTaR0r3ocoP;6k_UiW0l{^r%Pa`+ z`o>Wxw(b++V>GmL4E6%54qr>F{3_@hz+}Puj21DIE&+t-T2Kjbh${Ts=mhExd+nU2_k9& zq1E%`VGWrx_uS$?MBi%5&r;u0Ng-DM&tR2J>q02leA`y|xARQW{Po}}d5h-QpXCduB z38E#mAqO}B#N%N9gZ*TF#{O8ecE$G?u!U^eTg6q?mLeeUS}M3?^mR;Y$0^IVMx-+~ zg~j2Z@*yWx_b=eUw7pN7d?linO7BQ;9h@9Vm|>E_i@28Jw5y!TvK+UT?eAr|u81$y z+bljODqUG@x;eA?_Qs>LFS{xZrS>0t;jlu&_#vz#4fI8-kn)2lw&0ICb2~t|fc3JI z#=tEG;`jFyDNt@4%>Z}v^p+lobwp`YI}>L+z8GT5x%Z5scB5IJx+{fE{{DKdOYOSh z#X3%w&dqk&oic03e4;?-nSvR(L`qu9=-Ae!k<6!yA5HVHgAWfHBdtC+zRPzg4x$YZjIPJ(p(ECmAHXL2jR~$&@@*(5_4wcwQ?3xx*TY@q} z9bic&Rz5y;M5Grj_Iz~0RLckTSoT>qna933Uva-4){;m#zB9ASprx*qVk%;g40RKO zFDq_z;o=o3g8R=1rm60<#+K18(X_=-j>`fwK5c`riCrzOVxwaYEqPqF`{a`?$xqTF z11So%{&K8?1m+S&IP{nc%;mME!|86pj39w$arJ6cX)L!`B%S}AJvP8blIKURQUCQC z-0`*%y|;=E_nK>8xw49HfaBByv1}WW8CAduNjn#s!PZE!L{P!U7tpqM>s|x4x4zS| zG1PkVhhII_U)0Jo7-CFlGKVXT8MN>=$g#58?c>TjwX)Ya1*6FJe`s-INI{>vFQ|T{ zu@NI%=%m$%_+;051YQ~@+tzf3;@!P<+^p*cT&Pq#clV4ckMGRVE`ROq$(iu%U2Q92 zp6u`CRMEX_rpCVWH57DSqt3m07!6(Jwr7_zm=-1xZ8d{iW z*uY!BA6zzimMN*U_bxZ|RIRlbCzV+p9a?om>B&dEvS*$DqTyQ|2cMSta;m zGr_W;^A<7CXC2LsRDXp&DB-iMya9{KyT@JU;BId}H(QgN(tra1nB>C75s-2J{}eX5 z6$Oid8pY1ejt7LA4KTDE#DWn6Sb`qZu1~fhz7aE%QHbCtILdS9Qqt4ep@+}_eku!T zuiA^Q{T96@v<6af+c)`8Y$&-nMCw=p3~xNRTdFE{^Ae(Zc+XH%UGDycp z$=Eo76Dsk9q@)?BV+0=m@U*GeFp+9C@)u5oIGVgQQA!lAvK~cX=&XDihXO#}opIN( zaKjP(Nom|U%PVVMgMFDCeR(RRH|JCz-G6hBFdsi~YSG1biHN}~_6K-}64&ic$62Epy2A4sYJ^s`~raOPLS6H<0_ zE34R9)4|O3sI?C1arz6dh1ir#U6E>4Q4mf5$x_=o9t6U6x9C9sRESj7`Ie6BH^(4j zZWUge>Dk<-Af|msS>@2R*Tjk*$A&l_trsuce)^6&HHRrH^bDp8p;R%eh(tD5BO#XElj1y=1!t^WV-6T|>GKCbr7T!39u@vs1R2aCSA!@~a z^GNaRR@>_3(_j(+!|>eqb{y;NKJ?CE7zDAK_EJb_?7NTbsSK?DyIQkYuNUT&gCyySR0-&5Mi&+g=GNQWRXMbB;vEL*JZZ06@e$n%bM$*> zyp?!;Co`RHj4HU(xsmVs8SS56$x7V6U4ZgJ2k=cnRLnF?i4QbyuLVTpT~N>!DS?Ub zr$PgN{YTyzwcud=f66<#G(SX7&pj-LVIM&UT0_5d>gT&Zsx9(wI$DP8gXXlt~muO0zWUkIV>27 z6N`Wuaioz~lqx1T?jBw>uc5hE0w}#ePVj;Ig+Onfxk^XZ)d+j#d;)!O0Ypi=yjB2V zSmdu;-_)c5g|WGty?q|^MgD>y z@bueV&h8ZHbV3b`q=D38LIk~PE<32H$Mb^~^UwK5K3Vb`s-m_YT}SnMbKizm@j7Ua z6hJgDj;s$P*dyP@dN=w*HkyScc~bzcbsSyur8bi+Gy)A@gFoJ2xq_WExEd4Gf1)fsje=Iz8zmqJHIdLaNELpC4Aeq%Zy0)Bfdehjt09erHmet(~vN zE$Nlo)bp77pUz8#8u-dX!ujVFkimeQ!@idcNpXjmxv>xVJpF=ZRLKm)BVNuH1Xs zuyvKq!5YMw=ZO0rnu-dFi+#2P!iNH8p+5=zec{l9< zd5_B2Z|R@5I72T*>g;D`9BE!zhl?dts&aK4gFJ@E?TRf{p*K`dVwbXL8~3knc3o55 z3!9Y81c|UJJpr?FAi;b`=eXT6L%%a&lz~lOM3vQW-BMnQzE&7guN1T7b)w@{a_Qg= z3JV2?3RZnx^%$ZV$4MUzQ}Zw4wo`;}e@*=Upl!11S#QCI#{E$-kdpDwxO5d z*~us6%4b#oh!2oY-E;xmdS3$yTF@EZ3OKPDY8D3Z2E^n}yUm?LkIf(t9#q;t87i+) zB3QVVLy1P$hVXK9qsJz8EchM$a>bl0#k5zFXmcaBTVHPwBzZ9IkP0c8-L_v?e5){VrdYS5&n7FF^`_3^n+$s z*;)$S>-gXG46Kb@W|mY8c&lXUmRIIu{{^dmdgY;G=@|T~e5)z#gawr?B}p)HYYA9`6OfQTBaK8`5$BJV8sj zfHqlG$#D1DVz+5jAQJ{Mn<~92vifiS9<6>L9n)t|-p+fHRIrfIe_g+qz|j8rwP1PB z^NyF8QY~=cS@&NW0BEDE=l_$cuS7l;~-NoK6a__k58936(z{3G@Z|yQCB+PuQV-><4|A$6X z#|myy0Ho}J5!fOPG0>wSvN(rV;h0XXP4N6soi!8LGQd^r?YctpwLFyIxcCD#raE@- z|GTIJ8!xD9={GE~qi-cwApVqr;F|XOaxssWByn=!<%R38Z|Fw}nl4|m#u7{lHSZ>= z*tc3Ux)|?r7UHAJhFi7$eNUfi^s`G!c&O{*;RNaFdq6+H z6r~Fl0AkofOhfIE962|LAGG$aYa!@64ZT3h_;jhWxbHDuiD?(pe)=u_xwF<(F)vox z{KNF-*Kr!fg^li*;l?Su>AmrY43N}Z)7z^5IAOn5>rr?i?d8);yC+Y{yZvv5gXz0& z4q+YSfruyLr6EGYbV<>IX=Cc+u|>X^Lg~P2B$qNyLJsYzLFg5C07wGIt+6QV=@OUwx;|hH@7md zp*>(U7EdBVVg8&1Ge^$s>8x+qx#VLQT1P$l8eXEe7Z*z$*6mgLrc&DhQB5tHeX|g7 z0D*oyRS!OW0JR{qj|xl%aa-1WVd3(P)!=Y&o{hteM0nLr4n(x)%gq%SA%~SxxuCQ0 zPIanAX)eaZcKNMI_luOx$%(LAp@*Qfz7f=N;OPav4ho}N^X1@?g2qBHrqS`l|KuA?MKi za2r(YN%EpQ_K1BPGS?t~Iy{i|#O?fjdop2D08nGom8n?&h0y48j$yvv>I>rQrmWn{ z#ZO*&-PN3VGj!OMg4uG-)=}r^M3!fASLd7u_XO1T@-UE~14eHH4i=kg37Yh*&7Lv4 z6vz`sx_#rbFNxkC|F}qWFF(l2p>`_SM~!-{PD!1~^?}**bpB)+SE7xrl)Ts+w8MvJ z&DwKX(-D6RUa;gd0YXSkkg%8jO+2iBSf;Q%W8S*4+4HW?(!t_QK3`u-6~PZ8E(UTvbykix*N@;0$H) z&i#A@6-U=vQi75u-@NVK31aeg$h1g>gf`6xa(J|BF$ap-urgk;k4${ z8|l4r(}RoLDner)Z8u$3#i<`1G_jtEjjar5)Z^GwKGqQN7W*Q3rwYqh6uXC3wN(#5R^R&WiM1K2W zK9DtbUj-pOOqu@$^cxMm>)|E&=Q1L#n{rqL4;~SZ#A{H!0a6c0K_*xTSg&7Kf~_2} zMF94LgaI~*?&%^{ex74Lir|nMdrgl{85GKDu+>ZZydvi_#j9<7PhQ>L|7|KuozkaF zD%#FAx7jXS18besuGE%>+ZCM$_sgG}S=>1uXJ7eME9aa;TRsqG%KD2UN3Yv$72^%^ zKp60$=e{w981myQs{$`OEDU78ze9pOXskm-0yzpvwIC@KW!fb?vSjUcs zKMdN<`$lp$h>G+a1<*X(6LOnNg^vo?Q{`%Vip_%_Zbhc~cs+-Rx>$W@9tkN9;+fcE z3RCrJ6h&2}(ge#MT$=qv_wu6~v{eg-NdeJs!Ciz44o-qm#w z9C#7qKvFXe&#V&$86g218G5DaZ{NOEPb5QiEES0}3xfAs1`Nn9BBsX04o%Jy$6Bm~ zs8yJ1O$c3&DFjtt8t*AIo(;xvI&1cd6y^}ur0ba*DA$4H+)c2(@PKXBpTMD8q&QOjx(zzYRj~QYfFNP{Ng_(| z#_rIQoDJeyEhTNL0}+|VuHp6k$c5&$C^fB+xX#L~w}#)}=H^ zR0_eSnDc#FptF~1o%D>LrF)av=f3yO!z|CSqWDUk$#?s?pHnVNX(DKr{so7*8D5H1 z%b7FM7S5p(-3MEAbjpX7@Bl9SHu$h7!K#>E5OPT>?%wNG@VIGpSb|2I4eEO&^CioY zG4ONVYaCOUL3Ag6n+B) z^)za_5zLRq-59oHRZ0^X+#0)!26udb&7NkGNclbN{Fb(oo;TW~OWHbY_oh_qKl)PK zqpgE?g40ypU6P!lgRJCJwr<*afp(ugqJN7q(McYpkn782Ho|2G-iMZJ!KJp$7o6T7!}5pS5Awl#W_@-2_z+DF9~1@nnEUFDgc zU!BGlbYr~Q^J#nH;y6iqQUkULGkA&TYNI5o)4RHJB5!n}Y$yJ<8}hvR?)$K+iW|f) zX_w4d9}%0kRREe8gGHg??p^`&DBSy8VV*}O6r9lXv=hHS25@av!Og0=X-&m!Vs9DT zo!&JbDhp7I`%F8m3uj-jetz7C8)N)cU`Mf2DvfRTmcuJOw8w|uZOqQs_ax;1@aTo@ z^y>8luRh7Z*Z?sr;17bIr=?*nEGLNUtLjK;wRbc)CqNv;%BZ9Z(w~CU5Yw zAifXK+#%L@7;_4Iw1Q=4dJD)u6QR=8dC!fa3qHKIoO*MdVe&1@B-jO~ToxAN1gZ8Stf$BBPHV4z0~9LqV=JO-`I{wx<#^Of=L_iAnR)7ot#+d3)c+s9MDXgE9V zwh^Ev=>=JU=8p@1Yol|PNWf@s-KG9P6MOz1_L7I|hp8R_v*J%|l(HXHu&)CX1XZay zbP)L+UZwmM2CJyz`fS8YyeECsqXnibXtWGT_v6wC^)D7lK!+TM+LYGirWTWo% zxz>i@KN%Pw#W!yc79w=u1SflgPS~V=iQ7*P;78|cJOb56XZj`n!VEggjG0n=_;uu6 zfo&9KEf^ktY)=YHtl)?eWxppCEEFFv!yqtUUg-(}VgzIV_BB=l+pUjy z`I)G%FCEIofaZ?|kfWqZg(9iWcn#r;YFOj0;RU<aw}`-Iv}x!3!rbFUBCj9E#+gAr2c#*mp`n8hT*6P^5n5;!TgWhJqy zYp4B7RbjVTgukft_;$R_S$-M5%CaZRJIoqvS-0ALy?P-BfAK726fSol{)KKfNUsc+ z4OlgjvT{!4r2YAhRbSENb{n+;b5gzeky62o9r1~r_bOyg<_9`O)ktMc`5yJ3m{UTaJ70_xl(>e+-h5 zmcv1}g<ktUDdw88?>Kyx(-Vk0)#y%SR< zdv8FSMiDa1FJZoE)=jp3`ru>=y$DHv&9}WCHV_3qx5O`ZZvxu(rhmMm{?go!iwO)5 zyd~sxoA<6W7o4^7$?D&E0rrCWBY@-q^Zs8x9;+SpusDpSPD^V(VD<7HuXe7MzIx*) z++c(GyMH`uMzT>AzNe|4Bz%9ut8g;vM~6zSG}s%Zz_Ie<)A-C8>clOcMLkEbL$CB(_9(8wujNxUV}I=G#W zB(R=gBZCAe2=;;8djOSs9-lhq8GMVa9%DB1r+*i>yIY%SDPn3dC}372Pinh56ZQK^ zCjG>re%tF+1p1y>1&^(;qvq-(iQM$pEC@^Kwvs~|qZ-=bdQxMr^^W8(^_7eCCbWvC zWZXHv(+`ECbpAz5ltQPdlCDEty~STx9P94#nqxkqR6P{dcxLgwpS~+4^)E~e&4Rq# zG?inXMt15!Y$G){{hj@@98!K;@8*qnl|-@yWM^4_D;tm$GS7;L@QE) zp6^}?GpKa`eF2h%eFodqbkz+7$WPbs#m}tvln{s^QTPi-#nun>j_oQQGsViK`R?4o z)J3GZJ1wB0yR)Xl%Ajy0K60ef8fmP%GtWmCtWS6Qd8O~d32&~Mk z<()Lxx@!ao*x99^;m_YBQi;O!+Y?v#f2}=(pHK?Cm1KC^HkIxI?N(JyO=;EcHqcmg z4GkUOjl@(@X&@Tpg=XgcqYqnzZd9g^sV()7@gq-dVRth6uGNt46RnnH$OBoOmj zFRIxI&>vl!=gUWMZ99a)E(lJuFeX9$pu%)V*!|4~9wT(Jm2`dUv+dZEdW5?U)l!6H zjo$3U^@49vJdTyQzjh+zHxQxZDTz`nDJk3J&qT;yFGm7c=Lr2ZC=Vp!+~MmJMi2lyQ-FTvp(hO29N(=Qg#S|pAl?zd+mK>3Z+#~eOeJQ zwf*vB)5gXDwiAwmT&vTJBHJO1+z5_<%ot%~QvsStUS#7DqX{u;V2JV0gy?^sBc?xEy`6 zdUq12ZL#8)UHeSr9v_vvO?KZRZO|dUw2H=79%RA_4qEOF!Q)+K7s1S%a|9GS$;3ez-4nnkAxwDc1&#g(Fij-qe^?v844=Wl zl8p58LDo_j!IP&kU0EZ{v@QK^Wf~C4g&2y{a$@tD+ zd35&tAd^en55$3#%ar7QE~`MWxx?Ot7uAluRoHTws==nri~WKj!@m4$^_IuCy)c!@ z3Y0%+9+vQd8@(rLuk`51h^_zw;}`^+;2^W85kLviGT%&7S8Z2 z67J=na{{$MVmYXzg?AsO&V`5>@9Gsc4+RZF^Wv#n*V_B1^fKEz%SFxhjf;7+9h(*% zqL}U!F0G^OaQZX65-_a8_-yr)g{Q^3xOkWmHw9@Cwv!;P7yFwzWXpmw(fS>rfGw^u zh4$w3Nu4c2b>BDusJa!tY3&+X9p2Y%>Kr}^8(DnYplaQ-LHiFMckbsi;tbRa`QkcC zcbtzz!Ja)i2M+7GdXrp4{{>l!EpR&LKlpSUT1M>%ZdAL}{j^5tsEez>1)>@nG2SKw z(Pt2md5jAd*+GGZq{7BRV3`AfL zYgpQ(K()U&B&e8`>L)rGYLDDn^A$mp#fkUb))V@1PNnR_N^~Meg)+EkkIFnn9aR?8 zM{gm%Lx*DHmeqneykPP!AlMO4>$`B-pA-g#5OEK zIWX{)k;=f5>Rcj&wGQxg-3hH1tX|4>opv>N`zb@Nt3qq*ABR~Ti=*N&nM@|Ebx#Y1 z52$BoCxBH!28T=ofyOFTgKPN=KdPA$G@QzLrn$O)(&rooE0`%30^|2{W}svRG1Pls zkz_omR!VHXNNzUu-5ABCq37-RR|kfrTz(F(l90X12LI0f51kAoEFsf0HzB<;0WviF zyIfbN6A-!v(O&xBKvvW%^Q-tbED$%H>Vv4I!&gi1^x6~5IWN0hna}lLhEO4HA^j^<_MV%=Eo;U%zoQ zFKp#tBW0f2>Ek8aQ^x*)K=_`$JhOVrqk`zR9IOO{nqa`LD8V62#wel-1KGh9_YBxE zmgntn1dl6og_xK%fOZX}eAcIQxmHDOEpv9q0M*R8Yn339_BFhJUwq zFL56|4fz3Knb?Y9eaHq}#ws5?(p#Lc4Fe97!-ND#BAIrLw?zg~02RzR)9@Im zos$F}e?NFO z9q{XM@7~43BZrcbkuU%OR}h|pX06@sZAUe?|LyucK{;f6%LuCx;iz#1Qyn95bNDBLo{?A zI(dpN=h`irMmpaE#<&Y`US0@$6NX#}T7J7nY6^iMeKlJgt^$5j1w_q=vOlx3F;_vzZPRG&bTq?A||jop|Qkp|`zwO2jWcg+;!-PO`thuT4oi%}oU zL|#J(X9mfFIO~>2$1|@8n=W^0cQxxf#3`=ZT1Oi&N{U-IGft#o`FdExmC9>OQj)E= zmL@Kz`tH5295VAfJyaXD*XWrzd>CN^b3^#8=9VW?m}VRcHT};gp-Z-|MNQasxel6D zw_}*e45Q_G#YoDv$1;Z_Bss5>BMw3er3xouJeA zl(?hq7=WbFs!`IZZp2`NMsM_0at4A@o&D=?L!UT!oWIm?uS`@tlrJwe!;#43a@3tZ z*eo{?wJde^1`$i1HHAtUJPM-j`eRxMY1}x9MbeQj(s+>hd5B;6Lknx#^a9mof=VVo+ zTS3#)bA7yG;Ddpz|I~-bPzvjXV;I}s)`^YV0$B@{$dm`XDaTV^+xvU>d<)7lm4p{Q z9O46><24|RgFO_+1oC1xS5ie8!M{~gPK*lm`S5o@LX@Zjd;>adAtFr821)idS+FAk zTZL0$N!I*gv)nd%=ElmG;Hzoz!*6=?n>6~8)!cQMaFyS`PM7vPlmz1-G$Qj&G@)^Cn31P^nVe%D2dwMocg3 zKGu}gbk%Dm?5!-;rS%U7HrmPQ-O}GHG0O)xIY%A&;Zy&RXLlX?PxxNX+Z2gGdT&r* z3d)n&ONXhBf%1x^xuM!OCG~jR304yGj@pTh(+lsDlSBKL>Qk*zxzz$y_7-vwl`uc+u7(<({xdU4!r6mMlp(<#`2Lr z9?i~DFg*a9oOa_*?MNGu%zADE~>SF|P>SxUV8t+>jk9-ZgX#UQiviLr5h;~y}k@m6{4 zU`yQlulEy9?Rbb@!N)GuU0OAJns|?}9VS0z|H6t-6kEEcB)r&BTgNJ!RYC~kr9jvm z5#+h+;n)5gk84Ne8}$AK|>@4v5$C=if%D9Kp`1bXUt${s@M7$SY`h!H0S zbu6M01)mv``km*rIBev{!NSsvgo{AZHXEoXf#N~pwDn!4n340saSqhL$R0f1nGuE< zkstsPq9@HEZHaeU=EO&B*ywXo{eAjP+L{^Yz~uRy;@bb6{rzzUQ0MwqK}1OiMkNxGr)Wa2OGoQ z_&ubl0S#}9mT5#ncdjWL5L}SO_0kYwL~odUYldMx z2xQD*E;S+wg>b0{W8>rMrMn2So~FTtOzrBaz?N=rZzI7fP&!RjpSNv?2~{d_v8Pcp ze=~<$`Do=I5u#8UW?%SgTg&ayO}SgvKCJrcTJeYv=d;rIa`^U+mrynF6@AXBpV^Nj z!w1H+nZK;v=^ewg&*tkKbL*0wey(CkZ4YySC!;2I^vsAFAASrZ+02su4CJmtp z09YZ+o`$}2LQVei|7FlD?_?9|_KuU4EZ#9KJg}2e+?ycFQ`I@!6SqCAOq%m4ifrC3 zUw-bzIp+zzdOVM;~ykn_^jFF zGevy~YC%j3zWm{w9D@bDIJX~-*G|g`?cgoWv~=6gB{l4|Z02Qge3cHxFG^r@$CVJP zsLNuh;igN2xW##FPNT@ivk|`r&c2L{_oC>JTz4Uadh5c0#^1ANxjXBbe56oukLAUS z7fWE|$12RDGjEO-gZTNZHOP2gfl+mQcYQFp_s;9-1Sq<`HZ^UJtPltqgB93(7+Tua zV#4V5^BI#l1O_4A8d;;OvLVrkHj1D8lqr-KEq!qFf*o*ZnzPJm zx9U~~V%ea{ZexBPsbTh@Z6WRC{+=o?3#ry+5N!_WU+qjglQ0uA6tGYsKT$gaBerBF zW@dAc?IW@|0Cwag0)2c)!0;uHOLcg2A6d=ryBnrbt?)9AJh6emDhNwQWO_)N|9*b1 z+D>}v`NQN~bC}!<|3QR^D*6nmCCnXqs04%67)a4Fd~G2W*QNfM$&xu)9^H-aAEV|% z*SCOw{-|O|b>M{&@jKA$BUl2GABdTETR$5F zX}?p?h#8u}77xw&RYU@;HUbgFmmn8XN?8iwcgnv+Snbm zwsQK%;;{Xr};WRHG7SN`aK->6tfg-dIeCZ1%JmO$$_9AqouLefbk6 z^;+69HfxTKjcG-EzP=$@s3Kug&3G(W#x=l0{l53HpwXw&xZ-GTj|S?9mh#O@oV3-8 zk^S|XyBZ4|_`96^X%9JrRa9u?+vhG*qj|0s#Hg&WBt7M*0MK4LV|M^HB#` zPmZnMWrSP$StAY%o^@ zn)5EeI`$`0KwFNO0>KHLtRl}Zah^}p(T&2hnkyMYp^|RDKuyZcKbr^gy>a~P*K4bv zy;lDmd1>~j%T;c1yHysOJ>Ph8OHfd=NF{-41AGKy?Oq%TDaqTOf5sBC-nt--L*DyW zR9$BE75ubs9>3t6B7|C<++#a98&z=pwgF-fx$UZ$xoG$SP2pJ7u~#-9r{W%XaxJy;SsTnu5RD;g%h-C z)tj?Hj^~oS8c6#dHiqj3jk%iKjX$2$prn-$hYr%yj6IPc5B9e@TzF`ykuYg+#BLt< zk_Uup$n*{O=8Rib?w+G68mS2j8Itn%H* zWtsFF6~nO<6FZ+;XY#On6eOfRDsHk@cY<~e+QRj8{8gcSD34@IZ_|YDXk-Y&oBuml z6dl=Lf@pKzy;DuwdhX>#3EElxau;*RQ7>5^_LW%Sw?IMRBXw6y)woc{aohz^l>zrH zYpGT+`^>A$@?U}i&jZFuVJ0FREx<+4gZIEAU>3kWs{N2Mz!2e|Am+I~& zN=ryfcc-L+w3MV$-`vVM=id8#>yM9!d#`xceCHflKVO*1%n?9I+ysuM~l~o3` zetaA6+PtGn=G{ z>|qx;8^=}o3s>V`8)%FK2Cf2BFtKzw%cCFBbs}*>*}&1KfMmb!^r*?l+Sc!*1hRux z&!L?`b@pamJujeqfWdsuym0LB)UEH)_p-9TnW=lvpm$denEgU>B@KuUfNU^EIUecJ zi*&?x1;flgyZm^OA^a0kp@CM1&EX+t+`9&Ek97n$Ll@sCt?!Gwgi8~QLL)}eVkrR|cLJCI&5 zLN+Y|iLz}~d|=9h*#A$51+85|hS7fz?xPveTP6^q()rx)&<`;ZuGNX!y$( z%|_buLoO|PvKxgos&AukQ$SMSk%)S8C`7_%F=9tqW_i1=4%6}&h zJ(fKM7N*9tXGL_#Izp{lEp+i2|NBtXpwLkD3hU|j1&{1{cp@{A1T^@*+aXE7$#z_y zpqH(J-rIBmiLWxjTbRo`X`Mog9ApHs_{+NEBij}NhePZOhuWGFHV?ZyUv*$AEN2tT~uqishF4xs9Ar0Dh>Gt?QL+Z zd{2H8(pb>sfLBKh9g=$lPShD7;S?^Y2X}d-(iSEW#o8an0K z;v@E@mL1uzxj_2C8y!b3a&B&pwKvmM+O5LkM4qS;I+5wz%iH=NPS<};>`GhO&!W#( zwxH3KjU@Wr)QNmxr#fbOFg7bXd?a7M+cF78OS64v(x;x{dL1}hnM^fH7nA?*wm5P= zaaZ0c7kqS~KtB8gqX2c9pW*5(}pw|LJmQPNID8+_e@xqY9|Y2 zHnVz2K?_n+iJZiust}P0(-{~u2Yw466-EI8 zEg1M$fy^@?sYQ;Y1J4v!h~M0h9)htmf;%rxo_M<*n#1Vw|gId#Q?S?w^S@PUeWr$vQk9E+Yx(CLy+bOiVE zW})T5OWlg`()!RLmrGST(aJk2DW(;r=}JgL8l=J6BTMoTc|-BG?Gdw2Lb61^$jt<#+fF92nGRJ{H1hYIOU>ySxJcO*03_c@}zYw_#GYf5{I$%+<$qGV|#5|%o5!`nk2&o3{Tg7%MGc{TFOzwwMked(> z4pcKZnJh4Zr$AX)PzfEwu%fD&4msoD^eXq?FMPua>-OFta#%CWLp0l69*D#ydJyUf ztk!{yGUJZxC+jvZ?8uJBjlx!-b79X?f&c;txG#B9E5ig$>34x2^Nw!axydT!As8Q~ zp<5*|a?tVjqY;Ef_I{nXNM^9-;Wu=^LM%WKkJLhEHr-!4+FQ$k=`TUIQ4q7>NCs_a zpyH@hI2%r&`IHwITshuK4$zK!vqGp-;}UlhYt!<+#NUtg!r=*g)rku`qMDEzB30l| z1r}MSoR6k&tt^A2p#HAkjH_;mwN?ZkM`X_vYv;cpAtLb}t0>UU3aGda1jrUETQG+5*m8F*C=17nu9*-KYP868vXreJU}CKg3{*Au8w}cw zGF322LaB}P&_EuaRR9MA(x2^=8*-o*@~IEyDrX=S{AK`XA*sN5OYb;6(%`l|<&|u5 zbsW}UnF^!ONvsKa)p_Oppz?c-#o*D}C3bb=dzK%D{^l@63Q=ZqvmjOiL8vs8-$S4p zG=k}-U-1mrU1fd05wi3sK9t_3M&71BDu7rkkV^+xy*S0LpvR`M-OL$f9$!s z^fD5}EmgG~ELsnKuT?ctBAx{mD!0e|nXX09y0s(OL$ zyAK6ba^ObV9!{;Ea|wCEFgcmtBLgx8)Gx;k55;{%m>FWh63u=|YX|>3rFB(oY^^pJ zmVJo)>3)^#ugO}GZ`NH;@M`}3FJA!^L;!~}fiV+O+JodPtof9$us3aR#AT#2tHW53 z1;2f!TUBKyF0i(gd7Hv_avK%@@~%H%v!%0jU7d*;|EW4{JF0s!Tk}y>UF*9e8z?(m zpPmX-?fh`T_=)q8@w~S^3(dw7klB#VP6FEeCynAZQ);3|L4*Xu*pIZQZt*bpKw;JZ zIP1*ZnjUyMmOTZv*4y-NAKH&x#$aZMRLoZ#)9?GOcgAOv$9#j{ddzWMzu>HG#>4Nf z+JX(@#b)%d?1Ovv?om;TUB6s=AeDI!SNZ**RURtpBaMTcbpHYqTbdS$k6IDhWGrXj zlp2mOmt2szla0N~{E)_i@o5d(oDVwR{D()hX(KUY!7U~~+9Icowi>z{R~lS|9p3Ee5SD-S{?td|j<5;ZqSaC829%IJJ7OB^uD2jqtvchc5XZ z2hdl`WXS28-*RN~;Hdj@o3KxKi(#0VEaCd~zn_T9!y3dNo$Q~>bldTkhwO81saic> z?h1RG>5Go)7T(&{wj}9MT*h)zSZvLvj8BA%u;qNTxd@}G@Mfj&_%c7rieL%w|9h}f zk}*q3J?R~7qLj02+F`OMpl9p~jAI(W8nqyBt=NegrsdF$qQ**)%iX+p2d|tIL%gb; zhuP(7U0D3(Bs~^MuJgN@?rTDbMWBpdY&|U)OesnM9QmS(3cuP~VHiNc2}TxH<3$(J z89T|N{O=LIlDObDiyi4|`#c)c91{z3J8pQtt$lJi&3CMFp~+d$iU9wW3YR0h`6bWb z#EVIFa=yk==Lgo^eP6jJPi}$*hCkrHMgSt*Q&5o7(jw{X>hcHH3^W$q+@6bHl&j2^4$ zDuRdE3JoyHZW|b!22e(GWzy3`h7EYH46#e-t6lNwl=GG1W5mCR=6h%~Jz(bj`!wE} zSM`nm%+qXf2K8@C-VbD3L-n*SWJ!dy9eJSpMEsAhmKt^91F|U2ShqL~KRyn_=nO2W{G_?S*6L;{#R z>Ny1+mw$$7I($b>_K`WiG{Og?zH(Mptiu*OP+vHYRYFbtIwpqTv60dJ@L>P;n#->c zA*kdElrhmS6Vp{l2E%pH(0Ur1XHv+>Ctqxn9ozme;Z-|d#_j1H6G}`hEZ#{;U7G8%xhjFp-(_Cj)*Ywm>1oYFKlHP`gdjWI_KYBw&f*bLR(=jwqAW!K+ zaG}Z<$|hgAhuoP>EY6uncV*ff^$+pM6i+Ds`>+W3Fw7hL*U}ju!w@AWcROi(=^B^@ zdky6RSix}_O3={kq-{s>s&*FWORKpQ!)rkz(*~F@CzIqU(zO<~YFxbz^i|TSb&~<_X z0{p*cl-v1+xXCxAGp5tgYbT3UtPl-fm-8`DlJD%jE1{mO5OB|$R;WJ2Pq5a+-m2@% zLDAliqnnE7cSTB-uY^}s&}kzWc6SuG8o*461??HRNwM7B-M`MyUkZ`&hW?=;{2bwh z{2lONioMN+8<7`mePn{Rl*>QyB{ir%EVJ(HV#SF?CtMazlY~B>VEOj$J>xmme=g|YF1-^1LCj-; z5AT%^)Pae2T&|B5>Z2pLEk`lo9^N&}-NllkNzZOi#L1?d5<#R`-!V`UB7TcV(!Hcb7n` zLUHbj5B`4h^2M9`r#A#^3DMa-SV;68cBL5!EHE#-s{fC#qg@Nb>`M`)gj!Yv%p1Q4 zi%S`EMyr<>?O4#hd)vfJz!G!aW3nuI_`q(GqcJ4A^Lw3qe?^Z%#yks7U*oqf`TL15 zEc4HiyAP}PQ*|{m09ll2_YzA<5wz3ygH3wRIO%+KHV0{;YwmWA0?kmn1XMhbZeKt%|US zd7fhUTpvPbm3BIljpW3oi6xH@jT-Zeqb51u{sxlZZHEDAguAZ7NarE}qieWY_#r99 z-YXf@f$8w>p#E}BO^9kuRoBXDD*i;5a6Ljmg)ur_zznsj)K!)1W}xeF>85R>qUQhD zz}~=zx`4IVb|*{WS7?|wLOYhj*wnO(#LvH-X50l$BCVV=S1o2SgN8vXTdLynD+Bu& zjEa-=`iIQJ&KCWJ%gcN+T-ihpht!X+_tHO&{`Y(%Y?%dYC@3hf5wIn;fl|5HY=8y@ z4J{q!mMJmE$Q0tDTmVyWP7?(hnvXP$YOo;0l^e~yVSmK8nmRF-bLdLlKh?CO>r8F+ zUg;o?8N}#&ee%-gAXw7AQ{9`hR?}y5U&QytouUhAbvPZ-;pfk7nNW+28T$Qb^M#cLyt|4Bru%*JS?b|PEGTgb_s@bpqS=)RNg8;&E`d%Z98__a&NGf!f zzO!9n$Z})OzkFunQ{t0PA@dF6&UXescq_bXPkg7w>ulYQ=3Zur&z^OcI9P3kd24UO zZ@bZ3=h}C2_WyD;UR78jk>2`uW%KD4E=oNjK*`GESVwq?Q(uVm?X0_Mm8;Z~r|RhJ z*?!Z#*bb$@nBo@k&}wI~@qt-4MHM-uXzl;rjW>c?I9Uud=|@KEi&tZkK8oKjWV>s= zNZ@kur04T~D|U80nlir0J;_)|6T^u7ti{DXibNK7zE^Rt+@aR(xx@Nzu0!+pw5Trk zAIfRC;WH$(mFu5Ol)Y9lbY4`mu%<3BS?rAVXY~o0xbV@>T2lUiBCC;KLlvZq34oyS>41635gNUFUq^PbbS!E^4Ru< z(Mxu|n=f-73Uz4Oy6_U(yg-orA~L!6PsVXSM6UO(4Wyy3s0@G@=)6(6^gUQY$~c1C zlRN_cA(c^41n2IHl`nYEs9+l4$fb8o47r`VS^W%aTKlLyI*>IYDv4+@bUesM2OW$y z0vj|9wQdqVz1qftv+kYq@V^9I`61OBMhv)(G2&x%Ckj0L6EV9Qy8Dz#h-s$xEk$v% zbB?}RErnDz9W*|o6@HtO%1Rzo;Ya%++;sQa-zcZt6j7Eox4Mdfj*dP)KHg5^?ce^` z{5Y6fHL@osjVrfyw`UViBT`vck4ciW#J+%3zni3(s9@hSvbZJ^pLK#FG;vOf1PYly z2b>Xv-WO-5jEsz8Fi`nV=#BLV!uFyf9`5{zm&Vo;$Iw4pYQBNZVT$hkMm?vkxJ#Cc zo<@FYqaVZWuGHr*id9c_t!-)GdhhDV2L89|ild;80_zybEEPkbo;+`*qAAg2Cd0CN zD_b$bEE^h29TH7-Sw`dCkNI+aRdkBb3-LlU7ZuBU=#9Qej}@8e@c)z_GXD1|g(T!* z#Q+d!2(2y6I(Gr2T3olfwYD$7eUnRQY+#?LFCp9Nu`L#~e4T z-pIBV5Vaa^m!m6b#xni;zFOp|W>+bTdF?Xf+NdzcXqH=%05e+Mqd@LHiuh;1-N|TZ za3iD#4}EFYzPoyB7Yp#HPRQ$@89gkZ{g_IetVl|7UD@c{!j0DgwEG4F<+t$TOv!>x z@O7K#p5hp6_+RE9cl|HUkm5!0{23X;!@#-lEA(+iGGii`RI#M zn(JU=KAs*$-PNE9QLmEDi{8bK+B$Ti-zpHng&X zw%a)S%^9_lX)26c^`ZwxTa&ZYJWHd$NGbEfnvX2^k*9`ZkJ)V3aP@~ErJpV3 zS@5P4dEuCqvEVs6A(>wq@eeYgq@~9E?>kXqTvDJ^so`3i?6!`FOJcb84JWHi7|Q<< zo=4JGI!opZm%eDX8dpHY>~&T!YpNvV%y~ggP*}C|_sXXJ$!~}8+^pFN5zT6%AU?`Q zQyuJY^X51g5RHUqeD;3Ty34Ft&uw3iHR`q%{h{ZAE@7!5Wj#Z~xnp4JOi{PgwCFB8xI zRC7J2Fl%&DIGE=l!ayPH8sc+V_tfe{=4ldL4?bd8!RK}xy&WAii)nmraVIVIYdSlgd^f7`(e$^Qi(!N z*ia1v!H)SFKE82hJLJMm4P!Mp*&0j)%S6K}DzlV$vtJKpTzIh&*XGu=KKxp(SMScn zC)}|^SM^q!jc~qc#e@H<^b28934y4T!fVCCn13@xEz&6KU*KayQv=;qw6Y&RtV&^+ zrux{`S6(2o|}n% zD}vQF7l}JJiOC^6d{8iU({yQGOySOtyy4}c-a@dmtkC-QT29IfsgkJQN?c^o+KR}9)6`fJ4A;`?=8N8GE&9^ z+%oF9(V4CP;teX^!8a6MtZFj@H8%Cvq@#gDvGEbEF~N-BgPV6sc|=}VU`~;5);u-# zDaYvZsJK9&CfNP<7IE)>`Ce=Qj6xg9h!@y0JuW@F;-5Za@I#?dn{}Fx-~sk;R16=L zf0It*4FxCyZku91>5jkfRmO;(iV6WaHa|=b4;h1fmtRn@v7@6yiYdUKH7e#K(J=N# zKcR`nm{byB)=FIJc1(d@grp@0-{B`g8lH-xD)p?P`K>G|%=IbASna-j)IcD{t_Ir9 z$zjase?-TrITBW{yK{wHN4D?&1*rVOi0Wp&=50HOCvT&0U~WFNacRG^YbH!0Sd7ti z@WAhK6Xh~LNj-*~4=GeKCJ%kibF^SsSuJo{shnKTzKTIjm0=>klKfjr%H9S=+^dxj z--QzwZ7)4@=gE)~R0WqLkGCfq>>z@1Ztp>z|1Kd4jkX#!%}AbO_t(Mon!~1)@v~D= zx*dC0;p8*xOT5YcElt7_Mc=>NrMfIpyF~qK4?7VfNiQe`M}(e4$Z^jU3dK7EXKsur z$tjZOVanIoqk@Wy>pIv+MBKiJtDKR(4Y*?MucVBhPFNndY747xPq*^ zS_+Tr_Y?9_%4#p$TA{CRU);GE6KRvp_vAmYhkt|N*zxd}wvDXZ`FGq$pXH>Xv5*SR zm*TAOUO7Uqh#Y3menL%Km@B^dHbNg0p~P&Rf9D-2F5b~w7Hvzh-rQaHr|%2)QE9LV zx!>?PRV#k{=xT-$RJ=j6VRg;ffT;dCNWNA>f*0c7`T!81YEOcypN~lfiO0HSy|2wa ziTr&!UX`y>`6j}gPMQCOa#VXVO>?gSs;e?7(A3(4vmhW_5t!~j7k%{^ z#t(Y^qIwei>t6BS*U-v5hzblP^jBg{%F2$8364#B^meW0gJLOqpA}a~s~cXy)|Zob ztWNPpzUqY9O9t3gNJwj8hM^>?nC+m;jzw9>_N|)l*~XfiMyxI7^6-+v?u&u%oH$`^zqM)7=R?|sQZcco64k& z!4LVAd8%@%>MQRp6MnUPijMW6qFiw_D8rpfClPrgm_EA0SHrnpw;>ra)ZS|&nKSly z?F(vTalR>$*gGX7VPiWVLUy&MaXUw-UTpE#4;j4IB4}hw?g&#Y; zgxe&qfmAj?G8Ai-*Sx*;xdrPM4_W;Ee+~Zk|8M;H|Ca?09ioQ=qk6l-GoK`B^Y}$2 zg^1^8!tPa&1@+=Gn)tO_#Rp5o{noJq+p}T4t*1e=Z}?#{$e*L5#|k@3^!fM8y(yw$ zAm0a+PpCJE&n7tlJoB_}hS)H2g`FBI-L4>Dmh-uI;;m!Y^)3i)4;EpXla6SD3hp!& zz(xm;08YQ|HyyW-pDnWV_)>_n?&BDYC>SO0ImTh+-x`R zzj+PRm!{6n)zE{U?(SxwAqRtNH!Ty>mGbg(v)MXu+fE71Np{5mSfXD;L26OX$lgZ| zUrL$#!`N$NX5`xmt8lAn18@0DE79%7Dr(q6qA}QW>4qL&#<(-E>9)M9#t)IL0#WqwqolMnDp0utq8J`LUg(U0 z%o9gfS2uX(0JyxwyjH|u!;V0ZhM<8&gWq|J3Y+4NKab_80jLw5o_1bSo`1YVa@l7B z&Go5^L;ibiN`gwJBHNu9&V)4ep5Xd&g6LSIu}c0Qf<%6>5ozwvIiv78@!1N}h@~k# z{_U03zfuMpOQ(UtYySoGC-9zR-2Dsy*LiCw;$8c1+InO$ZDK+XzW2{gk6m! zZ-_}rHIs{W_oPYs-GXV2aLtcFOv!IMhXSOdh`2aEm@sui)bj?=p>6^h2(#eB`}d7N zF@VeV0@|4oj22^LViGquX95i^DpaMwMMisYXZ!WQ5g|Ul0kG2BJ37Rbl&&cz^1p=f zUoe>REASCsYv;EDQ|1;hUGR>Xu0;soMl+8ey}q})Xdl`cvukzdVEuP>Z*^j9$N(j4 z+eq*YyiA(N{{z;sP9LBswiHJEn(|{t zTq7j(1(CqLR)+g;RHt8S6+iYdK7O*k?^5wr*o_z9czZ!|Xga$*_~PIG)7)K>p2tog z-qU7SO;@q2F2|BR$Ap$<)#4I8O_k6&oQTav5gEMF!t74TNuk`K^Iz!AMnVUmh3VHh zd_Xp9Wr1w(@$&MP)fJSM;`?EfUql2Gb2cvDfbivh_zdy6H%SPq6`;n~nVFeFdW8dK zw}5h!F@~o+zhq$SBPZL5>R-hSd$cy8?P^_tj%&&o?g&xYAnC`(F`Drs@s4wtk@{xL ztlfE^l9U!8qA-!)jIdOTUz~Xg7)@2Y^=rE%25iFc=;+tTa8*}VS5#8+Dkp~lT<$`^ zM*ChtNJvP&_04JX>W^=YVDNi;Pfvcq-Me??7pGQ!124+$u+u-6v&eDBr?dA1=D>EvP#;J zr#F`6XRPpnrKRsK11;0Rr~d#a>h>xL6R;6Q`#^oVlX5n-M=E|zNsa#O^bCR7`(j{i z?+}b?IURqDuf9o1p@d-N1Fz`(RHfY^$cVNER11I-o`@f^zGxiWJLJW z@jLH^V;;8@%R0LpS|7UYMl+=xA5KMP&3%-Q4eVprViu`34*2^6w=?5MX&0xQcTV+0 zx>v5bp`-h)>Q|T#wWcOKkCI6Hj5Mo9h$$*!1Miu>vb?mkwAF1&#PNv7p*Gq74H{4p zGS&aVSEsmW^CmqW@$HBd(puaDj3r*!O6Yj1O0KoDOd5&m(3zyf{u?Ss7~s`$tv#(1 z>26s!jYB~NRIZW<{O^B{!3K{&*w}2YYFk)XgpAD~zwf}cHY39*zxM%kdTUGPz$NwG z9;_f&ayb>Fg)6&pTm7SEYaTYDSOcSzUdPYhw#wUz>yV#J(}Uk_MjWMo`G z=;`T=pbuUWgV2Jxh=PtjyZw9et)SF!soAk04WHAat zP3TkK)8Sy0VdXul`8HvAx#qVhc3G|b-c(5*%6F-CsnaZP#y+{`wwN!B&y?rWN3q^^ zdS83Apf>NJ%Tul?;&w3_;|r1c^^to<^S%O2PDTLoWiII7B4)|&Zki*PyyYZ?;;t(= zyhZX)N|?Bv^iB}$q8SlXyyc52`=J2=?@r6=>qWiD{8|py#&8ft9!HKDV*%;@79?h7(9N7gJGp}WbUUG>&!la>!+pX=y^{s1Tt!;MD4c%b~h1U!8;u|jNZn&@b6I%L@qot3MwjiWOE{Q2o%Z-2*^nJ9e>1s zp8)U_xj7-c%!jf`g0H+%dZbfkL4b6rAA12KW*FJoeO2By&xeV6ZtCB_<-x%A9Axtk zj9SY{8(b63>EB)uy~e*HS$bpd0ZOnNS<+&jPv508F_Ui&nLkd-T?^!^FD&yr30ECk zUeykJfx~vUF3|eykb?ZQ7&BeMhhe?%bDttkL%T%yfi zU{qc}BzyigN9MF@YHA3t%uG6k|U*_|#F#*)apOVwIpp6G3}dozhufODh0ouaSaZGB4sQ zc-12rcx1~Dy!^Y9kl4Jzcs667;_Gtm`rZ_sv~pf;r%iVwvZkMT&8ylUljl>-si!^r zv+mpQqwl}EF8CvUgzUwsTa|Gy6nDD!cnil~qzl=W@3c;RbNg1X9T!XxorPT3#sqr* zAx_gAU&xp_YPK$dIlVVXf%*CQRZ}>h3lR|!*{@F!Kodr@*0l_iY10bucw+u%RFK^w zZ;W`xz-M@&VVi9R>x~NtXK4Su*T|g8RC!kf1bKk1FuF(h=LvRC##Be@pwOXZ81 z!6d7g3Wsc{b&=bm-uP8S(T=sohhMLLwnpf^k_gay>JAlUf^lz&|HD0^@OBhkp1Szk zk2!SLPw0$(WxdlZWxFBJ5+GCX4-{=jXDMlNw7-t5YSISGiabwTzb`E{!Ld9A11-hy zrGL!SPBt@b_Rh>uz*;|q#E}#5n(@iWj8C74A&SLwn_mZ);X^%YQd}Z}QU~-xLS%@q8yoC)*N5aqKV9Emo4uV|S&DOaD@uQ?Ye{t5TGTA4 z{{4hs|*=nH*)|C*qEX1Jgh`PJ; zCzB3l=yB;pW`k!5VFIe{q5T>aesq?6>zx~ywpS&Qy)!n+65po(U+$u23evPX4Pt2QTdF?uATm;J+ zrf}W98PHDmvVYjZK!zMclD;eF2|br7kXWAE*?s%|9kwz*?H&zA{Qfm41_J$aw~G(3 zU!|&&4VOx5<+A3^@;%GHG4+k`H(AD>FFwsV ze(>CPG<-R8uB0et_-q?Dx&HU_-|1N@gaxa(xfPvk-^ijLF>_0SQQF-#op z>#GHHb?M8%rEe{Z5I)U5?lU;vd_t;~RkF8BZ~UEL|(qD!P(D z=b)9Q5=#*Fp3B@wa(FG`lLKyPi@-Cp5me@CG8@vh56rfJfMWiORpE%ap*FW zq^BoM*LyM6>dWa;d=y@~@R0w|$vZ^nYVME3G`3EehBCa|c3IzvhD-Tv_4>kRt8bjb zDXVdq-H0>=TxeXzG$Q*}hN~D7FS@Cn&B$dn<9Szo9ST9mT+H{s6B2iK-&4@eAcOG3UdZL8UUmGcOtZd~9Z2iBxv>59ZTyqgW=Ch2^2BarD4i@5W{+P_9Z zq9sg-Z*N)nLKz>Ilr#`{nR3`{H)+x*iubns1}7-00(ESrRy%`~V+nAC`bH0qeFix!pd0l*x6dm_W7^}kLr)N71EzSxs)Z(e*2ZO`HKSab$!*}FlqAf+KN2dUKVc7 zxKf%-TF98#9Z_Svx(S}^+K(z5#$&oKX_mNg>p4HW4MY|$dhQL?W71}mACht#TAAOI zB`eD>3+sr6GN7NR%=5U`VgqzHw;V%ouPf-4x&}7G;$K0B!Vmc2-28k*Vq!2PK*b3XQmQyW2uT@!hCVbFhul5N zE^%)A%lE}nV=13p{fXUXCwcl>&XAs@I-25Tp;MSo`TME%n4e!9H$0?TJp;ZcetcHG zQ@M8h{3)l%Bqx6T4OcXR37TE(#L%tqxPgg}lt6f){`j@ziVEHz6ccw^R7)clxvVqp zhVkaQxVbe=PFA;Cf>VVLWI^R=8MwGSa@c_K%GRBEuiQ5gD-zXqtDLitDG>K?X=bVs zHHlYv=fA4)w}mH+q(=+ZjvBD?J|#l7@sMO^i>xw*D_9|$WtV#$+4{RVuf^ToHykK0 z66)1=F(YKp+H8rgn>8@#XYDwgt$%VJr|oE=Rg3H;j~5#`r5whAt`8eIKKIg1Xj@DL z=PNPEhWOOISMlX8%Tc-|sDfoRouC&zR*+|{FN$yao+e5g(m;+CWH4@Dkd zd~d(F-Cbcxfd=%1GQnAPrXy}e-6MkO69&?!*$6L{w1Q3l^T)L(2V7VHuaFEx`8f7z zhYc_udky!4(1Pcz_`Z~VeO}^Q$@Q<+e@M33vnLLk)_6w_uk&NRz3uifA<%rW`${3e zpHyZk7H58?NKNUoT*vsY%#cIQ=Or!qi~OGJ8VIoiuMWl!te=Zn>J5%6acsZNIepS| zmR|gaporw=nZQ^~0Pz~OzBK{MQg6NngsrIhp1bY4Mf_HkT-L2` z!IFT z!hk?jwgfm^I#{w!ixMD7vg#DcK2D>hEjE7oUV4R50`z|u${&)SR*CTyKo5hU@e0%|qaPc@HDL~ivc=7>Lb|a~{X(c0y%XU@!J*K%o zsd|`~Y`lm83Cg{SVL|B1v0!(X#^01u{^Ef6{ARu|_32K7vy(0>;RCkD#BbH#HpWSh z*x4ImKVCIfl>PgN?2+3zkVW8`_=3GUEi*F)SSyN~EaZDM%)(G;WW12ey3V4W_ZeO@ z69svxVSY3BEcBr`y1NM{0$|o$tY&9_bt_^R_Ddk_r2CEuJ!4K@wmCljWTf$;fx|EP zrQYyc{*hX|v(DuO{MV*vXV@G|RO7ikmw$R9J~obuUBe%@`}Xjo0HyEW8nm4l3VVdX z)2AU|4na*vhXUiRn!zTDI*?|@edoSA45k7QUbD)c2|#!{Csmqs8S#0RXS`f^KQ73xniaAs}t$+Tg&vB3jLxqb9>(<(pl0wT>bBTb1o z8EXiLBvU-6jAMkLIj0LF;hO`tos1!(~cnTMu z6INO6)x1X@^n205vzpD!F@A34enGYQ;`z{^*T+eWVYn7?1rzg)Pe5(`EkVbxgHDyn=w79NH25P8&DiaUt

{@&+Hm*M1*MjFkKRoU+toOAfCf_mLKN7p`qCxr!A=Y4^B6 z#k)PKV`B)@j3B`e8CV0Ag68bDzyBLd4>kJE=hs6*vwr+S>v6qv(WkSs`Q1iOX~QFA zbA0M_##TjLgeJY<@x2`0pQyAJ2LG$uS6f63-9CyHPotPZftA5kaM?4?28Is5B`ljrB-{L{;vYPH?2Q^X~sf$HI`rk`C{#Ek<}R{gt*c+|NWf{h*1kS z12(Mz%}+l?(j*lSfq4YsA3UeC>Vzw!+HMQ*ToUw8-QARiIBHhP!z27pr1W zaUl({R8hsa96k zLfqBiz{LEGPO*B-o48&>#?A5f`6@q3$V*FKs@s>6DyjB-EDYVl4gNM2^>=srIj_}O zE8gp6$}I?>DWdtwJwtKk72M&kU}PWntAB;Ln0B>3viN?^k~mW?XHAs7Wsz(3i!mq8 zsv5OJ1unQ-tmlPYY|R=vN(pmdF1)^~l_8vT~lB&M@_Ya?=^;{%hwoI)HmR>U2+_P=EHD490sV&1@hbUN^F zvz3$x01as1NEYhV7MXMrF53(Gr<-l14XqS`0od;9$kny6w`XK-Qc{bHp6WSM##cV5 zLMK_wP5*(chZc&)JEfuYYxvITvrfw+uI~r7^|3{~Gi+HD{$a!CZYy=T>_AF&2edJ{ zxw&E=)bqCrUz}Y1U#cK*_Spg(=R(&FFgf_TxyfN_1$Rv)`WY#20h_`6goGetAfp05 z*0KYdN%i7-YKh(Zx*4XT3a0cKBIu*st`Rz|bPp2uSdTCRZVZZ|BQF;{Etjy5M8L|W zSc#mmIpbqy?{jYa`jY4?@24NDJ=yigdgJ@uY%kHn$TH+natC&;Q_wl5rKev6_rL~d z#H9|HVYUJ`5dv0WM&QnXRBwa=GY!sj4%&6mOq{q83$_?PJlcHQ7xeX{_Y-M%`k&Ep5knu_2l3=HIhR@b>go9G(NtPKUFMI4c*% z z#y29}9)shWhL7t*BYzLpGO(?#QN8Do^TBw4bBdL8Oc5(|P6AQRfbtfc3sAAKu`gp` z)viqe9tQ*<8hZK`ln!_ku$NLVco`h7r_%7Dnv4~8L@%FqUtYWU?omg3L&G@AkRLVI zC7|j&@H@d4V$rC~IA>JwV8H!@+^1CgY5pRRJYb+e?#L>dBCsw?bJr8UEzhc{^aTZk zQ}>)$uC2d370;)GM4#b4rEj}JZp`SGqdA8Y=AS2l+VcPN2o*9l8ePdoy0XT@hq=sb$a5r{$s)g# zgw^+ba7FsKW-}O4#`7b$7{riV(4vLCn+onD5~YJ~cCpnLt{)R+SF0V@8^Nh(yBY66 zDs)Ui_VzL?320M7j~yQ+kjrOg#Q~l_a1-#6iG&wFPkEJlr-I zxFzlCDmXYaqzq6m=v}^nKY+JPM&%J`o6?UKhpUM2)yvV!-VeIt(xM%*6uDe)?9 z0R--pzl1}43jytI>(3CS@4uCAH-Ke$vvF5^_1-mm0u_QF8Fa8ognx7NnUd- zyjW-%miwLU`CaP2rxxi0?RbDIz`||B)l*Im0~!FN>_C#jTe`t2d^d~xgp2v-pz*zj ze0aYdUMIzTlWR*a!N?OL1r`oXSDgNvKUC@SU}-f`;+W(axo*AeohyhY8v* zKW}gEDP{OVp17~#k-l=)!3&U+7|1v~$x71001HK3S@AyJI<71luKX*tM6+WKuF7F2 zMD-27@d`82-)nM#GamWu-Qc#4!@HdC4&t7b?7TNpIlru(CxgCnF#p!Ls{rE34gP0{ zbVkxz8GLDgppAe}0{9%nFdlIgkeqeV(_u1I#}W@?%!P_+C}U)mB`Lhyc#xYyR8?X9 zGD_ni=BoTbvazJkGCTgOi{9QTcR{9`cW0ONv5FIElG9B?HukLbme(^WX&P$`pMD4Z zy;M>*5E_EJRv?f}`%+_p`T_17vwPag_^^9JB|ajy;_rVM!ObQpmT&7_emMH;@?$l6 zjDj0JmPgM>qNY;-D4@fNxzAvse_W6G6nnv!T-FA8C`ebk-!JH(sD-#r%pC z$ck~%EPp|&sHfutGSHVFe6i&J-USPH1y8B~#=sHES5(BuGvnw82-oTQ^`4r=k5RJ~ zifh2oZ4@nZNxMaF;MauXkin!Qp`E5>{DOCHP zZl79R2AFiVXMXSVKijS*)u24b9JH9>dXZyYm-l|J;wa;PTUanrIy&+LD?dFw3hW#F zUO=pvina~#f5SQ!-Vmv+r;sUKCvK6UZrJ&`H}dyS|A)Ze$i|T}Zh+KqI}Ckvex>0< zYOyV!V2-ZN>li=NZ#lH{;X0896JFum?^YrHmU!=FpnSi6IO%yU8sym~ z;y2}n*gKWthUmLim+dd;+tS~u zepGIe?7Ai)k;*>@4<@|8fK@fkwFE5$1GPsc*NXcY<}&p=oz-%E$ct}f8j;C*tjs)w zaAI-ehL!M^{kY9x+oiQ{W@{$)q3RleBW;Ucqw+6>N7y7_Y5jdFe=x9XplyAAB_cT^ zk_^xMH`Si;6HPwZr#8C;&H8;Y01M3zR4!=N+-67fnC#B8?M1?ArzuIrCsTg?D}_)o;Ien7>Lb>E)={%Li3Je{@bq9Uhixz zr5-VU05!Ru4;Aqli>bCA_i1${-)Qn75%Y&mg~an%v^;ZuYFnFVSs|^d*(@9~$~6nT zR8M@J;)WF;$+uL#wkH|9#$&C{EOcjd2k+SOyCJtzEbr|LV)%RG1bd=2^=Wu>T-fpa z=Pm@w>&3{&!K1=B^2~ZnU!^Z3fZHS9b&%~ddYY$RTg^`nRO;jjAp`s&suC&SmZ(X3XNh+_S@+CB1XQB_^9A>jNu zHWVRCf{eN!?3c9rHpdDnLAdtM2ltDKUBk3k(7M9dB^S=1Bi$E!C4(B?pH!^}+9+j%qsxh}2DyTW+ z<{do$kFocT=dus~zdtf6BSiKH$=<8X?6S$oPRPoh5h^o62qjy}CbBo#D_i!+&I*xv zA7^!azrXu;|8x7R>!GX9`8m(`dAyJJalBs7<0hhmt@b>pwGiv&;Bw#D9rpr26ykn5 zc+y9Mc+Ml`gsRKy8n$f>*2s;`FD295zJ?c6jE?-+V*jqz+a%WwHxMP2jGP?BAk3@Y zp7Pop@(5!N0dHoG}|Q2z>wdEg~89fQ+8031p{V8 zbV~)1Fckj$%5HFb(#~b7;yqq)XYd!(8j@7i85h4g{r@5=F>DerHa$Sn4@>LJ2S6Zd zIYY|>A65Tz6Qse&nm6L!!spP@l?h|OjMBn*O{rh4J!0~_dO=W1;T!5}c$u%Ul_)2! z(Atk$sGXTllUMV4qCf%tHi}TdHlIQQF%1E2;TeS9l+V1zydOXF(n2lrOS?@DVw;}H z-sbaG`?o=AV5qk!XI{a^1jWFs*x2hj7?t=!zSysi;-WJ^1m^hWQ&m+}*B$og{32YY zsSQkTtpG`}-Gw2Ctu^Mm*_EAbY*Hg5;_@*n_G)LR6^sR5meK5x5x9gO= z{|v3?cT-WGMoRvWdq%{CqSLI7sr*+CSl&$d5pSw}peIW7>payX(U(10nXD}ZLCH;N zwzE)vZvHQNW@aG=5br~m|F%mPd;@7LMeD~rk=J|Gr~kw~(93^m9$bSBEv*iY1(V0E z^NEf;%Y@tzOB~6M3h#T%gS8){#>k@4|KGYIc2LEDk`knGu&dv(#%0CkRi^zwvu8VU z*|sL5(=g6420K!>Z?t?qtATC%_*EEWYJ$S@Ic!Z(7M}6dKV;zGYWbgQ5^+K}3h3fPDC?{;GC9gli%{jjZIyM!rO-&MGAu-~zcwI^{Y zxm7|Rr{r;7D#5Py}R+R z4lW=FXHb}DZX>Yy%g@~IG(I@^`|Rb**R3+Yx5(UcfwF7MK3?;40WUSZ)V%yFaXJsW z`z(>eEgXhGEk8zn_vq}Y-6_#_XR5_3{RHN>cQXpUB!8UxvT`u=M({oh#LIW$%zp!z z00n#&Bua@Q;iSaGGtde_8b%Wn6I&C`y^>j)@yxJylFwfjCCl=v70A5uh5DJ1K3c$- zla3Q$9cDg>r^2>AmQw4oc}IzMA_mW!p6|-s&ecV+%V83#C$#hrYPp4(Ztk;-PmH1? zhJqmBI=qs8C*z#I!T+}S!%4LYtIeqzG9f!EP=ypiGK!qGW!&e!77=N>^<=Vk7v!(7 zb*ee+kr_P4>LAUfLB6V0=9do!t)P0@wOeDl%L-3~vIL9B%QrkfuVo$>C$pX=31-N? zUFv++@a@YUf?0u`j#2twPF`e2;>hd`B`EQuAAGsSbu`fV@tbgj!U4Zzs=s34XU8Z~ z$c4D~a{ZD?JxjM^2IJO|Dn5Os4*2Y5@;)tb^Y0oHV9fh=EB8h;wCW!|=>DXmuB&A| z_irVdwX>(Co z_P`+GY;I3oes;aTVVO#O&~!gQ^%Jae1C{Cj$&1Xg*gvEKGX)(=DPLMzI;g>f(e8X{ z6`vWm_j7ieqO9)XmA99NlA#?syJ2asHn6&Tuvf49_#9!Mx79))`kBxov7DRFdF+M* z&N5wDo$Zy7-&(qgCD2s)2L^T|Kz{>O&*)M3eweIB9qnnBU!@e?1+0?638^2#Ok&IN!Cl5U zBjFw}dq}kZgdy|wu-wx3*lV+)iA_M$QG(Y0|6cgH`6JplZGf3|Z{7a+)9D%$6O>-4 zp~VHqLl01KE4_wvb>j=88TVx&-$!a}3OCSgt;7f2dVcz6Gu3YNkd0;Gl_u9bQiJj) zaiF^uvbeHwGNb~KQuxPM(@%5t(dBL@g?*3Z=VE82I^3X?jR(%U(J!!XB0p^j2vuT# z(w**jo#|;9p)4UN9%1$c32;pxAY~sHq0;Vi_tzIgLlVC0%c@5lhn}D5bN&3JVYU4w zlsy9#WOlU-I=f__e$0W5H8a0DVQe-Te_h_}ht8U3r|k%jo& zLx0z%s=7_V%rmzw_9-$UvQlkp*eun(w;gcnp%v%2^%MV|*Rj@Xo6u{{A)UqBrr*sH z%70OQ*gc`>ahu}3_OHvD2eTbZj;eP%r4BgP)GSwJ&{iDpP!o-)4Db*ZA$cz9IOSVJU`3MW<_P%VN5)d%hY7824wXC%!V=n zyNUkr7y=nV;6Oir{#?~l_Wc41V=_?>ZYT$#P;PE+Usqf*Gc!>rmz91@&`CZE4GryE zQ%w{OFlh>;1G79*L{kI#@5#~)0B${+DE~Bd)@65k))LJR{34rl+M$rkDgKRjuk6&`P1F&Ids!>_P1p~-Afu{%?Z5mU*Wh-KjN@ZQ zIae#cWTSY1pa|E{)KT9Ef{j`JQHQIhWzN?wT|dse>g|^D22jurU)6nK-A`uZtjd&& zG(zRofA3K&A$#`VsYK&IR@lX8b!ltdUQPk!|5G>re`wEuO+({zV8X8V`GQ^^3HT+J z;V=Vo)8H3GOh}=l@YAPf+Brxj#RGF2ihYQTL23)=e3{3OZ^C#i=hg~PoRW!o^25#h zTK(7=sl!5v8)}QqfZm#!nR&`*(KGXcDT&+DTT&7Ykq!X=6$2A<7WBN#tgKRZ?p#F1 zgJcd_16mv%2uW5?3&3SY!X}SDdgS(MkaCDM#$kmuX0Mg=YC(CfRPkW4f(v2Ax0bu)7o)PM1z-tO`f>*M=M3Vv&lWT^Okzh?ahqPF=mRI*_B zu1$h7;|^+S`YWMGy7)7r0zMPrzpIE>0Lm4{z9#}O$m9~r1EvJLii~^)K7x7RKQ|#w zb@(#_ENAdUpreql3Yw@$zMs_IfYQD3pmA8Q?ehyV?HyHwX+3ox@(EY(@puu~AVA3Y5rag*#J36rcNL_oA@lglxUOD{! zx!`$3@gn%8Bi^Ox!JiKplxg2T6P+j)OIWvsljay}uCPnKT0V7~$X0D#F7ywQS z!MT8=>nU@{0ZcCQq;_3NUgpmaVjo(7Wv<3d?6Hy&w;{X_Q#+r7Dm+6mPU+Q; zDKStDVY_$hG7 zxa*hqWMuZbrMZQU-o#LUjo z3OZv%EC#761)RpsyX6K)5wy;>&-8R%|T9weW@0l1t`26Uk1MkDQMfXHnN=lp>BYf2I zS4q!}dS#(jFq6G{=e4H%3BbDT2_lBHL`2RbN-{F8bj-vc>c5t?)FKlvu7rMZ%c*JR z7uuh;m-%hy&Tszws9OpuXr8sFj7A5is>c?#7|Re^Vu2RVMo#v*`+pNN>yR z69ds|8%K}hlk(pafE7zGsCxT20QK~7jvKVKPd%(WR6T^P9K>0Oc1G4)q5Wr)gJDE9 zmM-~|a0ocxi-G5B18$9^-gEzDZboVx`EpH`x-|KW;A4JX14E*OUozX7$Y%`Y*>4#w zFaFSe=0PZUBl9tENBmFa|?~ zdK|X*9a2jk&f!;^nFyuVNVh!#+I!Ad9tLYn-R^ypl-UjG)BaQ8&Y3iAD2S# zGFj)f7FEzDe)yN+!{_$`DaiIQ0m0Y8R!~jvp5Vz4;x2yw_o8Y%Y^3k#FR9ne z7mqs>>F-OOF8E;y@T}ubi0nqQsGqzepQ5JC59QCy6LY+1uS#(#2kXZ1Buf<;Dn&ad zOuQ=p(y!ZXto729&u72ozHPYY>+TMPU4~gt8Yebcm#w*C)ej?@M$R8Sd;)#w1_~if zzTGXtMNUdLw^d^Ac+-aT!>=>6_4uInS5X_XcOScDZO{HQP0@>&nPnaPn38}UV|I2H zg#yD~?!8avKw9;*pum1lTMK*zw^dZIP&aPgoP{Im(8z=e0mhGS#jP$4Xo#y7X8(Zj z3DNAQY7Cu5^f$~7mvU!hWC-+BdH?ujbLs;Qr+G0$c#3KUQdo)j3Q-RtH73h6yv-3Cxgda?@$ zTme%Nzh}hY;NYt9v`_bRK|xlk(~hUSe5+n^GN?L9IqCce>t=y_0vrHM67!~xRLA38 zzmL&a>6PbAPd-8aEW=B+yR;MkeH&O zknO$}7NbZ{O@;9Hg*f9D*aQfvsBiER6!V2(Og`n1YVjynN)7B;WVjRZ26nm7C`dgaX(w9b&45Wi-)k&2X(({-sQ zV#@L6n)~;UEtD}%MSajz@mHSZqlpih0<95bN!Yvi)|jeHg2OMA+Civ(DzM7^%ix|r zqRz=-i|cA2#3+DhoC+u>!@jAug_dTuY{!w0|ceb1o`1l-<8tVp7sLNQw0 z*AqACUNyFl>b5N3FWt|AvJzd$gD~1Mp-{VF&{P^2zJmfM z>MA|`IrwJLPfq`T+q;ZLxZ;8{hmrTm8Z-d6jI~%2AEkm&2RXzZt}>`2f{~r22$ha6 zn?XJE`_fB8E9)wU#5)f^~~*ERDi&S=zeR*MbHAw-~La{yoBzruLY(qUj{iK z;%l)HI5R#wBt`s7;71d5D5~^=b2$&<;})V|_0`eR3INp(Jt#ZD--D3HXgp@nmiIgn z^B0UO;>X6p;d$`o0%Vg~PDsWL_6a131p+A&o`6Fv4QC*q<866_hybVogfCZU)*j_~ zGG>>g%P^lV?P3gDBpPaCBW#`>-#KH%%5g5S_G`=)ib3gz90AVb&`{3yZ8pDh!P}fi zp#zUnmR6bi8B-X){_1ns!5}cUn3FQj#7XzgJQ&-cSOVYW1F6m4KEC@Q>=u}UuKyiG znl0kk36u4UG{d&P3G8_d!EUDu$TKj)YOVc7gj_}DofMp}E@o-6!V!xQ%u7<%)~tcJ zkl+u# zRk9O@Ad$kgc=}UBj-FIflKtl01h_` zp2UW}botr2IgNQ3HRRd^@+EEOzpT! z*L9`>g5~yw#Th_P=f)#|BhSUu{wfe?jaM%$kNjfk^7?*G_%J-*(Ma>>!!Bde=YDZM zmiz_D;E3P)_AAxHR12re$S$<(^g_M6SL^)P9)pwoU-Nz?n{L^$1~V}j^`-Rm^zs%V zMTIIcOhv;|k#le4sB6;@XfY34RM3wKBHG-=l&O)d>0BE}tZg4O@#beEV&~f!o3J`Y zljsgMD&*JJY2blg-mb`_Rws7+i5x|QDH*Ta6zIXRK1C%hpNUOSA1lAf_sJ947Iu^K z#`Urhaz*YHt=MH7+h>l51{gve_2mTX0zkftE|L05ov&B-t&i}sjI6Lq8 z^K*!L88%4E$;uL>>bcB0_>0rQ@|*6THsRygKtVnwPos{-yQGo2YV1(c+Ya74=22|8@FXl0N)QNow5tMycvI=Vm5Vjff}Lg zr$cE148Q5=73n-}WMvfZ)r|ZeMc_PQ{j0+K=Nogn0iSfP{r81 z(!up>R~8-sgRhxU%R4?UZMbL9;sX^gg70E7AO6YnOHBLuG{KC5Fh~&Cm8d9Het!PF zDmUBzX-~mWLCDty)HN!Jl2WB&vw1d%ta?BpmUZTz&fB|{kXw3}w4r84nm5CmEbAub z@?meUW0_9#|3F6S$}C>}4KuFpbmk&HH+R{8Dn!{N4D{ECeAqj?D4w5sUCGTCw#}CU ztwzsUTBs9`TNyod{_7@xW@eUT>g`CF^y(>@`@9^Y-7bmz;(s&mko=S~}VM>e%R zt;Uk4lu-1G#!}+EUA*=^$C*1+)yLkhlTADX-eXTM-Jej?vq|Sjp^11~S zFs*Mv{`9AIMzH5@CwlKP0KdqM^J{H@fesQ{rL#GFyE9ptk_R>m!7Sc!2m{jN#%Jgp z0m2CI+}pD)8@Ic}t>*ko%I1$-59~Hgmj@nVk%+ooNmgZRI19Upi64YQ5X8A++gue4 z|Gfvfcvs-FC#cZ9B>%F8zo*_s{wmVQ{Z)a`+{idTgL9~T#4Vp!iQlE2S&+_O*|-&3>;Lr zH%rjm!?zHjg^pQpZljHp{NGAycI2J1L6*@T3-#HC<~n(?#<$cMc~#`s-zWIC9B{c^MgN?4()56Z!t1d$ zvnp+Btw?|YLNEcOIht*cL;(&hEBGD{v0Z_Hs*iy{f_{~Y>N>vet2n&oHj zw7ZKq_`BysJhszUk2!7-FIkvBs#G!QK)VM)fdH>{BtNZ%_MV|>Vtwx1(RBf z`oD6~_S%Iw=8*l})TN{ygpkeX@#?*Hyc^+_$14Emdp&8)F#nZ{`A%p+O!SiB(i*D* z!OnVgdF3I;a);vA+@pJl>p%R|P(8tg#$5MeNu|BIZnO6S2WMT~8gxW%Zi@Np7vbb#!{A8)DKgcs#1@qw-8F)!uh4ERmD6 z`YpCJG6ls#UZ#&X&_c`bCF@F6=RYMS*AI8J zf4^CEC=r;_Deb^xE3{aE16HPp`o3Wt^536=6{c_r=>c_i0Z78p7XRWj$2XnKi5KbV z42W&=$7B^L07{xcS581n%Y{hPE9L*b!f)C0S9$&5_kwvGIecyFHag@-n;bAOgmB^&vXj-?IAbw5dB^Egk1uf9$&MC~j4JabPlq zfEav(Y@pciU+kf~@vfYvDxH*Ge2rQ7K-@nraJ?RV-B|E3=il|~MF*C$F{rbHL7oTB zGzdxM7!pw@dFYOK&0OXsW2veW%)DYCj%#R64ruo4z0ux>E$?2?7|Nw(=U#ZsU@ ztvK%2Wd9~cfj9>*)`;`1gb)CD2;gQ7=<7>wekA=pvhB6gBc_>rLlMy4JpjBDqUTKW zdcTbqpfDejJvXr$Uz>dnQzyJ(sBJYSP&9Ni$zz5kB5G3p~WFc0-UD?7$Vo>k-WI2Nt@${W=W5XAt zgNAPgm)Kl(e7pYZm$66d+W&SHf$0SRaWXSEZ-;(n@J3{cS|j6a&l}!<@=DZKk|h*s zPRIwNwccp?xFBzNuffW{OAm`uNWzTj^h$eG?BJ?1%N1;PFGo=Wff8dYqL~$}jK&AG zm_(77){A7p0MvZ|C!#`atCu}2taag=3@tEuF(NgEzv;h%EPF(A^}Aw|CM=jU0D}OO zHkkkJ{lH&-Ko5}l_Tu0)8WbdNsPzw=&l4?dnDf?WCra_hl+UOLh;L7q5g)Ry%=WYZ zysLHb?Afb$m&a@&Pk*A7O<7uD7D0 z9m9mE8H&r-zYTXziCh4;K}rs@zCftXQEsIN6kc0IAMhhLA*_c5FepVQ16u&OZR1c8w7vCbh128E zt&y`TEKKfysJlZfi;zKKe9r z;=+yZYg;al&X;t27*3uZDdD@mYh)1d;%`g1D+r`>5W7Xfb2M3@ewqS_n_8Ax7<6p+ z>XA;gppO}JqD66R9XSUnbfOm=aHvDxbKpdJyHQpjss{KOx2}GAZD=g1V}!f_g-*Mf zaYrB1q-54?cHD^eBiYSTABTz{>Gc=Et5O6QF8FVP!5y~U`YFy4hRw(= z#b_-u&>g*b@lU+=mP@ya*bnOtJ~McK)1Zd{g~P4u+L$wU=6d`V`qf=-p?PSo;9?{eI50j$!e%e^Qymg z?72+K%&ZtE@;5<>)a6@~UU>oF1jmbrX0t_R`1eli`(}jNZI);d0jf(h)Rmj4oU!XOVnodAGw8*$$jP)V&)mV4yi3|ILajT)a(O znp~l|xz9kK9j4-AiZq9JZFk-yLU81#D_-|UddA3FBn%mXnmM=t-%-Nc-HQRwPiWwd zwJ>^c_wRG@WiA+tqH0ud0yMm*lSk}FA06Pm^Hv=Jz5<=);nKc$ua~RoT0s!^Ot-z- zJ#7>2U%dyzEkyO?(@=x>U<7MQ{TjUJ) z@a^EfbK)tLa28|obvLKERc;1g2oOZAG;@=hf zouLyu2HLMmZ+VfnEgCBQ>?`n(8-Zwqe58L+5TaYR9L{~4m5zM+-OSvXiTpYk>B(F&tA2qh3x1y z;dL4(USVM(PYk4H*Dqt+$6fM0sT6xOcxjLP?VpFyc1>wYt-ZTjw$A6oVWHy+*g1em zh!i8B*d16^CGwOKFAf@{_YrC#TrXJ_{z9F7}%U7#_E3{<%GB&e2I zx2B&A-k|_Kxd$`){dsjphP9aI#||35CF%QuABv5$w~BPgxz0Yf zrzE}gg(bw5?CA<+lqq9R6X-e%%HX3It2RkyjzvK)BGS7ULMO$_+H25WswY`QB zPP@=V)7VLS6;B#KR5oNg7+z*eI-FYh?ScI|tG@Q1EyOxr$&>j68o*d64`}snjww}4 z!U(~wxlk@C;^x&};9|sfGn&2X^Ze_Em(tdD z#Qi^!TN3uUni(vqeiybmlUFvd6$aGQ;sf6d>RUKne15K*>fG$fNCmr)P6?bV1Eo@P zcf|gzu1qUF*ts}3kdhYgRoBIg$3G}j!hDFUVq!ID@GeDWeLKQv>i9&?ic_H2qVemA z-$OAPtIHNysQTAviJt(hOrHk9M$dJNs>KCg&VrT{c>^aUK-09?(_pzlg?nGhwMI(+ z`R8MtvUv6iq0-;1tZ_aPkz(fzbgwDAMG;v=611j4Z z5Anb3_Mvzv=s*Ruur$`rMnfMiH4}bXA(CAKb0d=YzLh_F^-`cji}m@{nE`SJo|!y* zH3Pfy$!|HGW%o}ewjKD3ILN^JTckY>_GTs+3P3T4)0q!yOp2_>zIm4IVN#x_n3|d~ zgTIsX49P*^Psg(KT5@t~OxZ~=w50ZCu2#J8bo8H4K|u?`UQtoeS5Z+=vONb~qsf61 zAPFFoFractfoK}v#ygLn%2Bf38qm*hf_E#`!~E^+u<^=c*#nOOlWGY%C3w>YiGNqR z39h)JW-P~X!%Jr`{JaEWL|HMXkjr)gy6{kANyECHD}D{FJgOa0hEiR50^y5J@<=RK z3lWKfO4sTV+c!4MqSWJ38ILIbU0mA@xHw_AAHlC*6C=VyzK31qA8khR^v|P~!AXXW zfI)APjc??5=Dx_&KzrJH-mN8>=|Let-~A3NiAZ@qo1$9w+~Zit2?Yd*Jh@^;&Y^D6 zG0)7Bq3~N#E_S~iTn%$_ZaqA0Ud2*eHSUhBj{5FdVeeyg`KMwuLClk!_B_qZ*_FYb znMWk{acYqFIgN88Z)VzcDFWF~8%$Ee{SX;Kk&YCH-sgHy63niy>gB9K8U1U9A}Oc> zAGAj??z?7+4JE%jyU#zHF0IM+ugv}A?ikfFAaf4WO)|R_a~#8CLWL@8Kc%2BqH z2EW%Hk!;pca8gi~VO23j@;sSjRQ$gS-q&q2fjD`Uo0|YIn#-e~7_qybb4D9ChMI<9O{F^Lu|@>h#DYT0d69ro`T5v+?RfxSu1G z%CYcL6>5AF7!}L1nqT`rkdPp#woA9mMAbvj0m1QtP5GX_zG=s67DW3BlcLWFl~GDG z$3Qm_H0Y)<&gUM0^}sKh3V<~j-3?pa;A_YB3xb+RS=pWUhE zl(*7XiLuT7CZgjn)o%PHSqArV=w`PZ{17}0#6}%MSJh)R3#=4+o+BsV!tu|`<3MHy z0~YU_%*7QI5X>nKk=0q<%O1;DsmNl<*SOQu(^_9vC)++&>aP@b4Yd+Rwry%p7T#)>w3($f9K3X z$#3-j{OZTc?x9+iv*s0ycIdXp?|7_6%sS2IUTDH*Z2^2AmuI z{{B~CARh?aoMf-jr(rpqd^z;8{(!n=ZbXrvd~f=@b{66uc&E~$P*-fJXh)P+{!}d@)JtJaXY)(lo?V;aEFkLCQb2Os-DNfap*|rn*bA@l8eYTz=;jnGcOdqPlp=^KF4QF*mMe-1fkid`@>z!}E$qt5v znDf3;40;<-e>#JjJ?ya%_}p7Y42m@OwtrZ=UTI@5*Yp-t95cp?n}~XU7{ZcDptE-ze+v+#zt5{#irA*RG6wskWQdWw0l+%~? z$x8Gr_|F-U_X@&DXfp!>*2pM9yJ#te&Y4GN%7-FXnV=n;9c|C+4yQ7~GK7Xn8Rt{Uiju=~Z2w}1-?zV0# z?rL~x5?tCrd0<+F^9A5Lxz;ivg*bK7X2i?}pth7}QF>N~jGEsq>c=Xql0RmsUAV{X z!`Weyj(lnW6p}#l3b-Y?Ba)`uNeSaMn zyOUS_UhIodt15f*XDIj9;UQ%R>zB41F9rmeUVV*yIuQA1PxNJjdr*9T;rm6e&HJz4 zH&YDmJlGegXj%@lnq3WczmD6mCSlK)`SCmP{Zrbi?e<-m563{C7MP;OJi9P>$w+1W zYxmiwRZnv*mi`@6``m}WZL}1vjfmm%SD&a7(idSDTm=?h&-MI*{G8XrkXD?tU(&ZI z_Nel2{MJ+D*=zl0kNNNvDR}cz_B8l`pY(~KX0|fz1Rk8Y=VqEQ{0>9f27oO=2IB?W zh>*mqMZr8Mc4P8uf6Y6|nsQOfEFqnu?{ah;5)@XC`#`V`ZveOluPr4=I`xpxtmaiF z#whNwlF*KPtt%y1L+P`9gq)!5jj4`LL;uNLG?eP@5zWcC7>6@*G1k_I)|b9`_Oj>& za5sQ%sg?R!ZT4lL#w%rY1xeA}?~6|7US79n)c5T+tRz1o?Rp*vhXX=^I~0`epKZj+ z*5mQ>^MUu8F!yEES=TQAVS8KMg_?U>OJ1?si?a%gXa?;tIMV&oe#|4v#T6=ylBntQ z<it<{S+%GQ{Sx%vA0 zn=tJod{Jko=2enn&?n!~*=#ubAa)IMPI_LU=U3||Q?E==>!qeoFRyyW#$014xJC|c*&G{eNlev`rLa(tZPo5l9@IGZoQ$ojVIK)W=QFWi59H)% zHPP8+Lh6=~7lizMX3wQAF1ejVdOs|d6x*B=7u2-lih#)d%7c~`fC@oUqO*nsDXQY!ACo;(wEvqaM39|uFArr#Wce%9b#3Zl)9eC?_ zbI8>uMl4>Wtj@NxkP?qne*7W4Is!CarsAy`yEPsQskadj_aCdcb>rU$H+NZ7ZA|UO zXxlt}=uPl7SJhRSSe7`} ze8RHu*i}#w*~oXe{(FxVUcK_B0#zg4>LtN$JANGldjZ|*#PKFQTjM3KDDykL5zT?o z_E&RHIxXh~ULm)IKI#f$0yWc47{^0lpb-F$?y>f{jv^wHwJD7jv&9Up;=0mPJ*)CP z9VZpEDgB!Bw9ac>(9<{%kE}iaCb*>-zF1hg8WxPuOyvE{R$wux&Ds?(6!5hrHdTJpC&QtCjL~Yc1!L&AxHt_wT`3I2Ew0?;GbmK`o6l2VNZBGQABCA>*U7h*$ zmWwqGtPN0(7`WCHW#d9I9!ozkV^KCeg+H%npO}t5XPLDqB!AAJKX4UVA(&iCGj-ZU z6ZiV&<|(kDyL92X+R{M7)`jYUiN(6Xl9xtPXq^x4c(3ph)C5$BPvH-`A5V2eI>Mg7VXd6>LP_YP@ASdzkNG^W?^<{lXQ&Uv2tCewl%K~{=O5;9Pz0798{+0 zwR_=>rXd%g#5gvk_7q~@<>-OMW9^A*O{TP*4?FKqQ%7MFSJz31`inj5Ilbl;-y9r2 z@4-mh4~z+s7Z$1*Z)4hi@v7QP!lx|x(6j%Z(qda_{micqa8J{+u=yYW z#MAX8K}T3E^wz6?1Jw74Qd*1v?6lM2kkF25e$cZ$OVptxkA7S5F=+P-s4HM+IVoSl zkaNYCs!P%Si2-Ln>#Im$=)qBOGrF+-xUe#RWbKt9l`)(-R*h*#aOSUu912gDWs2g& z_3zJV92)smR!KjVjBXxhBu-MXP0`GR+XyICw!mEYP7Y0QD32HvwrnS-|= zOnNLb``7FwLdFs2JLd0=^RQuRY**iR9cGYhx>2_-d}jZCB`)=CF;=3UV5>yno*Yb1 zy50++B`jGgRfezi7QZvDOq_Ov=17Uw!fmc#Sv@<=z3kY*YUb~8ABH=n*nH|PauRFM4>Pp>f`5DBSi&#f zbS-Il>$sPfF{-sjyb76+S9VsqM~V^^dw*WsW8$;EZ_R$I-II#-3&3(G>&u`vp#DMN zD@H2PV;~n}6v373LpFcp#&30<7Y|AB=ZeSqV>G`{S>_|T!&%GOheg>`1AR#9KbJUV zi$^a!{B!$$4VQ?#JlO;V!ylgif2W|I2%syit{ekDieD3ZUKLaz7g+VrL&&Iur&y0f zQkd|rc4o4F@#QMX%UVeeE3&zKplE{bYOAp-UM4GL?gFf z!}+j_5P+Xhdjz<+Mt68M>_y%&3J777V}1fXy=$$7t~px1k%WKF2tN5>;`$TZGCXHI zS?7=UGqnW&C6+^Yf3}~}M`GwE>vMh~)#IhJU)Nsw{=PaP684Jg&D*=I#0SUiePp7W zBCl;S^nGPHJ%+F8`(~Jt-r@>bCYdq)s)Y9Rv>^|ou9S+pmERK70V{#gQLi6fJO5_X2cCI$uAgOX)H)U@vk3e#F*syRB4pY zA?G=T>33ZbPDUXe#gLDx4>A-()d`1l)}=9iwNlRt2Mz>CP8_X;5~Y5gQo4pOudkZa z3xEWfLDb)ub@8(3Z7_G&mD8pf;Y+5zVu>+FI9H=Cyd+Hd@5u)1roE{^N}H-$6*zg= zDQg;9)+&KW*B{isGd6{5LuPzQY16nRRJ@d2N8f@B8ZZ7Cgg?zkgA`yr#XNghgS!ff z#i(Lk$T(7pvNv-0ODcJEcJ}doZE!h?f6}UB>BMtSwj63as{KqtW&7>UoMzgB1li}T zbWI*!F0wZcEb?LF3(sA=+qQ7vHb)>T=oK)VRVSWw?!5jQD|OTB(?qrD0n7p4W1Kbo zq{vvIpKh(ihO~X}upw*DEOjj1f97mR%U>|AqxXTt`kI9t>2%z}RE_S&dqWwiL#<3% z^??1iF>(IERVKmZlpl&G?tZ)AW0&`yrD?Siv#FJl0Q>P!54r^%hM-`%)N*!*M6>T-=77td*%(dH z90;@(6{Hrr+Y|ekeQ$F7zS)v}#{|OfYD;*7SPCh6^K@0wFUI())t%Wqq=+Y@CNzoxTX;?+soWu4YO*JC?D~9q!&??oxPG(m#2d zt6vQkK;0ebuU^CWZ90?5axxcY!nAzpe$2;Zo2?eqSQMb4Dh2E_M;7nfgq4OJ7w))V za83l9O2KQNi4k6EXSq5~t?XT;*y&GjyStW=Q8&LeQRYdD`89JxKLtaz#$g4f0&esX zqSNL*e_z+wsv8F@*K~j46th-Cl1xvp*x(Px<{=NaB)t2ZLl-Ju^zD;kKO5ZgotgB> z{MKTAC*&Md&mD>$VzWytkuQX$yd$qrNuhb+osa#t&(W0Qfas;Q}>~%9< z@`;$e+5FrH?Y;AK?OYd8Gm9G`{x5Bs_s#FrJRRTvfT!3|eF;r{_T@6y9)tl}Ed+_(U;gc*GZURb%Yn#Ue|nksD#TkTm2yR-HJp2}Q#ou4*wsCu{sg}9QcJELA36#gi2 z&qbbcHfmPeWVi^kH+dxt6pvkH7z)CLJNcZ5TsFI-oY33{IO&XI3^V2{k{pPQH&@Y( z;~qi?eO_R*U1t}*M;F(g<1!8ZyUao}n;QHpgb9>oK@%-#tIv-HpKw8%{Xb zvx}AywGsTy@cw?S(u!6Xr-e`e6WnWr64q&@`RZEVn_H*KIX&z;xu&>^IdP{Z_+a+} zM3syBY&G<*X!IK!@?R!Tggf)DAW`2xLQ%~zZNaFqxs~yP+i-`%Hz6ede$4adrb^nG zqmR_fM>?tR(_GEcxx|~QAy*pYM4NS&O4V4>fmy$yPL6B~ zRCXB&12)dCnK^N0+pe}P*(9*CvIYOFtY!zBse$`a&AwauNO;gdB5yyz?%NtE`_%#= zpNYT|XXEEJ$_vUTfJcxi@hfR0*%Q@EPm;H>`qr3O!?YTjyzsbmGWfd}$mfh{0cUyg2BdDy^=@-MX(Sem~4gXpi@1M7w zJuokai)s~vZpqHS=jUg(^Rokw-@lbJuP`G2N-m;e*s!wZ?h@-;w@*e>@I4F81;N3e ze|wd-K9!%6Hqg5hnlNC3i?A%60?neZV+$yML1~}G23x?t}tq9 zyj5c%WecF1R(W#SdolM`1NvQG<+E)r{~qtiyKs&rRF5hAP=c<-;kM;=iCv}W_Pz~^N@XGP;mfo2NPyENW;zor zEOLLwx_?UBvwBQrW*~t%QqTF2>g|H3>wVH6H?D;9^;O>W^)5SNR*{+dG9+@Rt*bjI zx`snoFfU@eMtbX>0}=E0K{5x}I~O z{~5b`wwoY&?7cy8D(;T=%qQvI&e#Qv!EOGgU}#lle#qRM`k;3tmiZ++Htor^hcc@w zw6P>c%~~8L!kwuj}NhhngGv?*4iiS%N{R2IDMJnyjoRrq_0yi`mg zmW>4$^ID0&aqmS3>(Nq;eRDf5X#8M~U!tp1)5_k9E_UL!+m@qB5DFMT)>mZnD|qOT z;uP0J^kv7M|A-%|h+nXn9s5D=PVCYCBNHuXibnp6F0C!?=pBa*ZI!7<%5{Tm=hZ(R z;lz|Czgah)RT)*dceA{)M$7n~NF|-nZ!8VYb@`tGHI|1HPdoxdwXBZ#${>Kp3BOHq z+*7&S$SIR*zxF|kpA}+8+Nx=Gwrsp0yV$vH^5%$~57`Un>^>TOwc#@ux{Tr8iuJ1V zN{k>D{7v4M7h|v$R8(3(0zd#$8KAesASg(}8iNg831k{)OBiJkxbb7k!$60R_yS-? zkDQAOKQT5YXim0&6!$&=XDgW3Sw8tbc_NB9%n{_akln;n7;Oi7m=})oEhhc%@4#J& z`TZJnZWv{YVtYiavoUdMEN!>HB%(_+jQAUlV{PW47l24q$m~CWSA&^4QQLhykVH%Xt4nec3;wz?<>RwTeJeBd(N?euFDA<1oUpXXT5#i!*VvmMum9d0>$YB( zhvg2g(SvM~BVi3ES&_dnEv?8@D&x5(>lLIE4E;${z&Kb=l=GmjUcZio%%DVu*cn#2 znLvB+UY$>k(KI-Wk>9zl4mQA4p|553#^Ccu#wu{t9?*jU(vQcqUGOJqIrmFIY6yXiX#EvAwICOQ$Zk z`sW@Z(#`Hx(kYRlGs=GW;uXyj@q4Y+8~x6k^kVz@Mp~gUjU?hd&pcm{(8gUI9lxQo zN(PVH8t1wutL54032;fvrhW$EQ6d9YVQgX}7~Ho9R3&DCa4QW)ybRaje8W&2(vUi_ zB4~ypJPFj*xK1AHL1wrQFKNUmfF~?tX~_&Eh4kVGOdOJ{Ft`?6L8BGUOcVmv7${i5 z2;mm#(&39ApaA%Tj?D3bE^)Re40+2=RN@S>DWMEFmmN#8 zC=U5a-O3+sTz`jQ7mJ~ z09}|?ijIPbvsrO!eUGH0mHK|cYd8l&9T|8rfz|dgPEEJWF7b2H^|xuZ@OB=+uK^Yp zfarTbd1UV8BqJm6|M2$KQB}2D`|t(}P)a~S1Ox>Y1Oyb3R8r~MbV!GENrM8?ASx{& zAhqc(0qI5o2}vpGQcChS7y6uY-rpJD_`W~h@$E4fJU(vsUh7`>J?AyA8P_}Yz<6ie z@FJ$82RL#!w?ls}vQd{JU%thgB^JatE&dsQ9@7nES2K)29K+;B!iBctnlER2k= zsb{CBzqf>4MR~#@q*!8l(|Q~9vs|I!Zm{xMuCzMH-B^s2$?1Be$osF1RwClE@E5Ek zjKYeGWv3lZP2)XqJKBj!aHcF@m$O>KPhZ=zT-AO)sHvSSU3yVAZJ1nz$BFLV1h>VK ziuY5>v4>mBOvYULhCbU03PGz>@;Tn()1&$g45@YF?P~|s$Wut46wfhz{8hYQM5atp z1l=C@HL&cBvs|u}LLVKvgzKWfmhA2^1Te*CP{a&EtLKO2+S=NlDJJEd2iYwA876(0 z@b1HylCm-`kW=|ITIEP^6nK$!AzLr&2(tLdFbSLB1iy5K!h&ZnZR;Ic%G?Ey_YRqK z`UkKQOChJM2v>YXWvW=2mHVIizsHrZ`Voj#W@~lVmBx+0fqGM7@b!%2tukEf175K4 z^3l7akHm$J$}`sv+UpVI%XkTKdBha*8hw-aMlP%JO2woCoXY4x2ZOVjY7iOp+)kAD zfd>6Gn5J|B1)X{!@84g^ScdV&JyR9-!7|}VqYkrOR5ugl`wtO8)V4Ml#Bk^SN{{rL zH*Xb7&RPinh=`B3SfQ0A%UfSX@xGFM%vD2JQY*w%G}@q02{O6k5vN0+aq>UB@hG%; zl&Sgq*N$qzJ@yoCz3s)>y~*Dx9(373+wzY)yBci+-D@`<#@)_Pjunr#3BE-cTCSf# zUU~czg#S@adp>x#ZRJx{`x!wT{fpOVZ!(?tR}Fjg+C7Jsp63E~pF&?!(dNro&8uZR zcSJR}FRUq^{-i*GQuf__)vnSfc=zI;&m(L07AKR->_f&mS#7Uq4u1Ql0~IOU%hA!% zJ$hj2lZU%4dRG+24pSaO$fkO}nL+avX^Etf>Pq%6(35>pm$To@E2oG;-DrEvF_@#u zqM#D#LUUK{s^vYiC~nf;{*~HIZ^h)rR+QhFhzy@kNpa0q^GbC56yGRAN{SbYkr~Js zW(X$VmT1WiIVUFiuJCE17qCC*H($BAI8>$Ieiz^tE!*OypLk5$f*MNh*abgHtT+z1 zx<;#NM#L@75;ixjDRMqfE4axUje1@6-`~O6G1~8P7~{9LfVEKW{Dn!#-UkY$7<^OFS-gr-RAw^T(utA7-mh zt#)sZW%_hHZ-z31pwXY;4Dpawa(R(97!4i_19KL#r>3W`@pZMfvQ;oIb1w4}Zas70 z+{ee)I{<`nHd^vDfD9s!jeBu38IgpFHxeh;TOO3%8THjl;x#O{Zrh)34=LZg+Fq;WW zQnDXVp?S)p(>Y0c`{MUYU$fa?TITbUA<6pJG{Apl7xq_At zRb<*HZWIS1;)|PoE8|HbfdzaNbaZDY&wrr+JRvNXSd?{}gZSICoCm782IZwp$}07_ z&5k>pzfNjwrRP;FG$zq&`Q{6a?QWKpk;t&9+)IpxK%1YST}qV#h=p|kog~R zh(0L^WB4?5FoZEGAh@<8iY>Welj`#2FU3!lFLT@BR6bkfEUPBMKCXe@SRL0+V}=?t z{7ZK}1ykP3bJ?~49c;l|4+jTMn5&|nCqOJ`o`evUZ=@epI;;F~1>)(l^Z1Kj@t(VD zk=Hnq=#p$6p~mcCsOW5V-H^#uVp(6?Vx(A?Z|M%2xu200wLSaGBM-s8DdHHl9+nB| zA^NJ4Jp=|@mkYQ>`4-S3mY-S|9u4-gej98o@hr)1PKyyPjn!|?rpqk~lsKKE`DNFTSvM2l{$nW&DRaWRsrYi}>TvZVV~Yp=Mnb?#`L@s^9pjngIg?DaTE zxoUbU6T%MoUno>89+4Cbfgk3u4t{jvV33Iq3JN-R?$R9`0C!&(6XN4TU_K-*06U~I5nwRX zdm$B`S!%_25Z8!7kG_`E3>kuJd*TZvm3Yta3;EuXl9xa4xi{{bG(>;z<^>~ukN1~k z7=$Xfw`;qSL5mOl_!hH6;8)X*aU0aQnSiccTxr5m2!+h%v>V!#O(Ia84z$fx8Q}-h z=BJ6=-L{E81biEqh}WQMGa^%9Os}ERC%?BA1#PdKO&0drnTCQX*noDhJUl#5o*FE^ zk&(*&i#1^25Ybe-h{lG7_s>?x{QUi~WO>P=)UrikDme;i3xPTCKA_ZCYaY*Sk9Xqq zxd%2j?5Iy5oa*G{1ReVE_0^y|gb0V4nKM~-IWs^_S6>q$evkd`FRT`cfNoX8)lqVY zwD8KqgW=Y2Z{2k`;&wXKt7IV}?Veb>b(kHiJ1-$LG5JKf^J?$HmIv$#C!1(SN^GLP0?s0 zS;BnU<=A8sD#9S`kGKO<3L8V)@s%&JXfBQ`)2CasYKtSpHXS1&lMg;v<=TPb=q{UO zOvFymq)Gi9+{lKG^tLMHz>4?YF2y?|l>!nR2%c3%DW%Is=L`DWLYsHgy~!!^yIQti zFI+XlnPdND<)G?3bSWLg|(B9WhVW%Iy0U06~k}wSgw5M0lj4MkmZtd}mdz{7rWZ6EWG0FmxE_V}?JQwzS~6L@&#usv_;&Wu>|8 zwUyf)x|xc%;xt(5E9;#T?d!Q6H+==d!j<2bc9rske0MdwT%Evv&O&@|f#w=wY{PKH zd1OxRI0$)Kg5P}!*?Ssg2egE!1uwhJlr{4T4a5jGl#ch$%+ad>U=vn!I!=A4H3Dz# zg&Y3jD^Heu1{PS0x_H$$x82u8H7aAX2B;m_@D2pDd;nLxH*llj2+I3uw!6)nnlp%p zQn_=88#Mo|zvH+EGzp_yfSJ*9I<$!TT0J z@zt-NSBT|I*8QNuzc@gmS1h^_}aP>zxg> zF!wWsLR7`N^iA*jp-kn3v5}YAi9Vz@OL${VcF2kDQx^#qzovb}qEa!Ocg%YXJqIgR zv^B11rW*A-!46=SA=?qoeH!F0JrQyXgYLsKI(8v5E#y-v55W{Tv9<42 zQ#N>b>pj~Z-_1HST{k-P^5j*z+|b@cjt9*(-&5^AP0@DdnU$_bAtM=+d?R7guvEb@ z3}^m<0k~1F{S6^Touk(0fL$nlsfPAcPaKadBgj-V1+V<_p0Ayl)d}s7A6?I~+x*(> zS1sAQ0ZEy9LRJBM!s}-^f(n+D_77+XwEdeBRzn&Snb>|%BNZvcs#o;T%pwbd*9^79 zMygEe16KRw_qlkqJ;X-?ZY%XAy;83qF24!hXD+efI4^Sv_j<|uE1S|F`6r(8V}7{e z=2lmH(0brKx_yTvt-Zwd?m2;H^!<{Q&z~G|8-s1@1^MOHp$gCL7`I63F#~(;c1~oX$2y1DIPYo3u=}I@G z>;syDHMI-HQ(l@ZO@+kKNHtJqmQ3WnQQjD`WJsG@ml=36vD3^p*)7&_Td-GA#8gr< zpUj@`-FofX*x!5R)0adSH>8Qpp-U7Wqb0Pk>81$hZw=&_I^8~|Y6`CXIX$v53#9mH z2S$sf-P8EK9a)oda;i6$&GLTsO{f$s-rzrQ6ZlFvqC%f5>yyD$G6`qLFnI37QzoBi zQPaT}=11QJ%`PAAKH*(5&+6p+BY1+UKr?4k@@Dj=YW5{*e$x%XTv!%H!n*X-NjdPQ zx3p9-3U^j6?-R$<$_UPUX8JF^3+H*}E!?;S)uRD)qayW2$f1bXplSDkyRX4lRXm?RKjA_vHpZ z7;bVmB%g4@(B0Ki|9w4X$N77ePE`jtlGUI~YLoq0rS^JrrT2U!wH7^^+1f;zC^A)m zKJQK3Mb~rJ)Oh1AAx{(YR?_dxkKQZn4;kKIp~$)QO+Z-yg#yVNLuy|$capCk0v!|jdUbKX^OFyPBPGG?cz)?@fNmFmXG%XbJF2 zS27B~fe0&7XUQhA>-pfRVuqr8(h6Fo0vdgeC{LjsFj}$C(uj@Ob>r=VG9QyyHWFxt zK%jH*d^+p0*D?~>VKP2{Y8FxLyvniCEjTbNz$tB_3&^Vt4|?eLqh zl+K>^HG8ss8llA5d)nwg2_jdG*wIW=+!x8&m9_jqFn6e!%|R7OrYVXn@f~!#_H37J zjB=%)wYPMLw{j&h8{L1jrC31Bsjt`e4!mRH1AXnAaQO>Av&3&fzsi&Xw z&U+)QO$+u7um;CjyZ2hLiQu-u6zB^pX`5^VLDVP=36Z%){+AHAnwSLzBnn#p31^gUO7~1CxOl zPfz&7FN0k`HY*+GoH!5Rjb&;Pk_0jPh^H7eYRRI86kla$w1lA+9s95I#1tMXM&*Kr z_|KYOuGw0uE`SBbP;t#%45vujr;|WawqMHFU!flJoMWb0WR7gj{^I(R&7+@kG6MoRCOZ#v4dS9B* zYcN(~Qm$=-qs>&k<9|UWtqB}@VPWXv0j6O6D?F+n>g`QyZ*%*}o{+@ZVhCc<1)UbLik>lK6j z`qWn=+2+pj7%@=<X)Sn-D`!Ct1{trGX8Nfp576(0~EqxiOKoWN?F!w3oCfQ2ls)lSREC{zDaPo7nGPf#f7NQeL&)5)V9o*Rz>g?heGakmHl zEVMRg48(DFyJ|MN5M^Tbawcu8aZ}*OLyJX}ef30ap@3w?$=`b8ORZ65b|li;HOT|+ zj0>E*U8??Qg@lJtEn!IhN9iOwcB@K_x|S%86FkSHw}h@CX~0*XW4A#&*;7v(NuBd7 z_d-hCb=TF=w3Hn!=1zbnhWU#l*O6;2Q4%k^-;Y#$l=~`9tx*ADAY_i#2=Sb)e(=zc zjy>tH*MzC%(WjYkR=SCEZo67x)x~pY0E0=e=M}#oM`y+Advo2F> zhr61CR&!ZeF1n(e@rO2i{1kjB5@+8><5~!BL`P>+WG|+-%3{1U3OsX|ntd*YeqZbL zjJtt^fpW3_3_UITYTj6t+B7?pBhQ(+YpQp`nX^n2F)-j}z{)=TjO8p!Bbdi8_t_@r z_AArNhzFfZWGF3oShy}>rPFE*Ijx1FnI=u4orMhJ^&ZBSY_pC#b$ct+YQ7sSdd9!o zoPn9-*tRky<$20i0lbR6j+jINe7)6JwAm9oy4I!r!%> zhTnTl6O26Un4BWEFZJ~41SB_7*G#b?*4@autbrVmvNexn*{F;?spO#aBck7nd&bYwB@ZsnkS1&XR1R(a<;7LX|#g(cb?+7 zFB{#+TE@5?MYJ^c;i`{O1%|qhdBJyY4838uUF%S$+=xhbq%5~v`&PsMRcya!U^JTXC0zUkxq zU!GPTHuDa0?R~03Czw*o7%tuauwO}SL6zX6P+!tlGCuUFN&Go$GlA&hOZ&BzUCkS^ zWYH@hW)|YEo%?OZT;jn+E#>-K>?DFRiP8KJQ(m_lD)^3-}Ly zmWps@E$Lv@K0!kIV{}OUz1NxxPkwds$5>+pzF=^sK$TBKZjP^lml($Na?&H8r!^hk z!7gr&8;<#H+sf*E0_<#-7S=K`@hq-P#{nYXHZCQwF}Oms!_QEg!H(P z9Ur~Lsk|}(ADKdW<4gT9h^ITTu$d6BU&Qlh$Fi`?#$JayCoATKSn9!P$RneG7L{Zsl#Z*&!^6 zx2B)qc)wJp@!wfrDQTL0*jLfaL;ew%9Mk)u=G^8XCDawYzN48)Ho5OK0zru>uDoge zvJ+hnM7)?{fDrY>$mQtp=C@MTt9x4p zTzB!FS1DiR`fba7Tj8VT+w|E9`5)eRs4N1WMg+#gtIv@hU3|@`@d^vG2J3|UH#GCN z->rV)T=KfdwI1Mg{#0Ux5!?9OebM9|7}&1$U~9RZ+uZ3fTiMSkCga~fFWoR(%G5u? zd&+Y%XqOa*ouW`OpZD&G)#MwoAti;+_H@?fd6UGqv?=3)D$qIZN1BzYtylk8MGQY) zIoZd!6W^CFd-fc1HHOy+UNjxh=-U9|eq<099jl$=K7Ze7&~*d*rJXSrCDUOJ!ch68Z{0@soll^) zt|_2T-mwOzLqlfgB2=LDsj@_>*0yJ`0leXpg*19fTv}MRoKsxz0f~{`(Mgp4bQ2aT z=lbQw7>_2%6+6T`rcPmV1m_q&L24*kPEs0K7Dau;z8Jyln>2DRuH8jDZ{JBxN66v4z9jTMLg~J(DXPXkh%ts#p4x z_oy>>XK$n!&r*yw5~&!A`kHp4;^f^Y?479piBDANnzF6?RQq&;;3}*UHlvusA#+dd zF47+0C*GR!_{<$Q3;vwLjR-CECGB52i<4(Rmw0vFW@pTXQWZw}MEMUFb{40}_U(V- z#SCoLHSkm2{wQ(#tpjKH?Y}zK?TYef32FD3F*h}E;+f@`MosaY9b+k)A2TD~Xl6ym z{{+9+f0a>{;N=h3d{u{BPIBYhJ}Z-+MEULJ!E9a;)4R@xmiKeU9%DAld4`84%U8^A zCqBcuwCAk2yTyzT2%k(Z@?Ss1QzbTJ8R$o?0pVbpOeXw3-;geY2d~^t{sD6#5^870 zB+9rh%FrssV$@9amXx-OO&Ju^JHTQB1{Ao{!VBPF>?iJI$o_|c)Vyb+Yfo!F)!utx z#zGeo0TY9lfxRY#JL{ZL#fr`4YCEy2#lZ>&MvEyhFO>M(Tn8C%_zXcEbV zUTQaLTu(uk+__iG=l}accZ~t2W|=*1DN}-+GJQAj-@p&kD;0lj8Z;2}G3EV-9%QFU zWD#r_8VIwoPSA=Cj-{b?I=DB;<#Gb`6@#sz#ZSzfx{p(i{+8ygyQEazQu&y?Mqj}{ zoDnJU;W57!Gm^>aFVo&oaUv~$<-18knnAd!0RBe}w!~*HK`UdYfglp6s7!e>@(*S_ z{!Ont5|f~&={=oAUNv8jDaD9jv8FLV4yCd0*4-*h3TG1w38Uf^8#1|&)7|)$6B*g` zWY6|g7@3PwZfCy5*&=6qK2itq2C4)60gnWqrq8bD##xfvKGHN<^xcrpBzB<1%dD54 zsCjxXEIg7hCGNNs3b#t=3s0GU?ZRt8G_gZGK{Mvd8!m@U(TptvD7;MLDXsg!zf8QO zomP+_9^6p&hpPLBvTG>I{$`h{Mn~kq+AV*8Fz!{)5-xP=0q)L|SKJ?qr3{s7hU!^^ zmDzbB?`Cy3wM^YeTr$i$^7I=?s4;jxsyEB>c2uY78yAYpPumN9FKgu5>zlgqsOkjF zsw#4^l!?w%B(JmQgw9AUBDTu{1irhbDl_+@XfB>Cz9pZdMH?Jyn2V`zYm>3 zg*H^hwI(@f#G#qvrGvqH^0Pbyc1nCeQ%be0@FIZlTfYt|op}RVHB0+3|K0`b-o6-d z3;A0t`{Os#3luho-d!pYG3=n2Kbpyq*IDX!HThN!B0%@|%Z{$;p5K`IM8s+&_VG(5 zRnw-aFSkV(nzwGD`M+1o&E{t>QGzuYQr_@5A_%7;%_lYmGgI4k0Uk9VMalG$W^cSc zN6GY8DU-N_=#-8@TL=wR($D@9*U05XGX`n4Cn}xw)(on{d?CT9BJ%HXRb-rI=cVpB z2dnV_V1CM>sJzsFf3=(A9HhdP3yrCbrkl|1O)<2KM z^yWn&;_4rFAEZlY#&PYRhRuRGay z`ddpx1myqw$In^<^-BGf);kS0J+y=>)NA1GL77=w04`a9BJV`v0@3#B=dx-i3;)Hp z#o|8sPrupxJPXs80#(SIu_G-aEVQ(YT-137s_+>$2p;Mo^=vwQ|9xoWA485xm-yK# zKXc=@DQ<2*=7L_7Yz9}R={Lv_cMH{RcXzkQlzR>9Fa_l(ymCGR9NgRAT8#S9m(8-- zQjOGk$CL3xv@88-gk+xVWmu&L1Szu{O#jdWmPn;~o6EV>^u=Shq2$k3v7orSkIUvb zDxqU!D%Zeboo#|z_@M8*UiL#DFcBL#4Jkesm5V=sGd?g4&NijhJVqku0_Owm`q6rU zhni7O?c#uZ1OhpaeN@8dR;(n05InWH%Y&$OlW3J^-|zHOa|L^EWS!sh(F_tCY`k%2 z%?9*om)R;@D9dgxXPSKd14tEBm&`T3v}7Wc$X``+GEBR33^Q(`nLh*~Y-*Btgrn(G z|5j2@G$SDojq>#l@*xsS-fnyz86Q3L9PZ^Bhr$N_d*^!*vIWWH)Wb<{_;b=@L}8*0 z@txZ$ZbA^W{weVPf@Rwec#5`YfNMD=eFp8Y$_G3VePBPlBcC)I`z`Q)ITv! zZE`Z0D{IW-d<{*N=LmLqH@}k)qZyao1xkWA5noSd#81W>k6-3g3NXFw)VV$2-@bEi z@-)ksUwumKdP^JAr?sx^wb8daKd3JqgDtrX3XJLB<9cT*om%UiUo7B~^YrfqUjRp* zqNc2nB6t#&eQ8H`V&}S*MYmmtTM5m6)rjQQePXAxtgPcNN-RABf2$LGw=fmG8}ya^ z*6!kMC7<|Djj`)cYp%aC@n=4z9P_8+F7pT}+TaY?l<8L!xE%FltS>+yL?m*?-WV0hgsD9l_GcP&m;|r? ztLekB*Jt7y;&;WZwCJRaal!fXoD|!w+MxPBNnAF5fixI)mryoi`ZVNLdj^}SXat4{ zjV@L__nD#*3)~%J*h9jT!?SXO9<+G>)6be-x_i8Zh|w>@;%aUx*FarT8&!*t-|-+^Fe;hkIldLH585^c2` zV1;!o6@ET5`iX}Iy5!bwhpPW|eswQ_5KZCdnDl?0S$);xSQNt$zZ~zGt*%)$Qz{s# zJnv4)x5+ zR%ukt3~ExIBU9(YKX>5uXW8hG@`LSbRWq%#4)ao{l%k=KvwkcSP`hgQvX?1?vY5+9 z&*rzG^p^gG`CL)8hThd%BET)L$rA2ps6QJ7}}pdJW1bE3ZMe>Im>FDFBN&$0V7Vzd`+?)qo3^pA6SbQr_xQ2xN-#P-BCsmKW2 zV!YCNrNs5#BaW$-Ii1P>@5g7M7c2DCz2|~jzXIY}$NZWt#kv-61xiVZnCUX2-dFpc zJ?3WMs}&jk*k%r&?5hiQG=Z@$7Z-2Ia8=`O?p9$_UYFqr4x$zXW+?6i<$K#FCmK== zbw_J^5yt}rk@!;Sksv9k3%sa_q-@$IsGVCe|DIZun<^K}X=^Ogae1hBr2aTrOqZng z;fGrsShYFY@565Arb`I8+rMI5NBGK`cu&w_fB$nxCO2LTp2(sR7th^bpCZL(ALlZ;X!Rz;ehOWMocO(Ja*)T<~ycDIvR3kPXe!J zA3-)_G_^9ZX?Te^tor-uYLz<^xbHN%sBA?yeQF|OTlpcv_J78?BsBgRq>Gu)fQ?c8 z_XVo`xj_HmH`hP%@Fkb-uOx2XX#31r@8=g}t1vXx_gvFzKQLp*X}55WqJO)w+&Co(&7$Xordn-xf9fSM=|5Xkk&V4J7gwI5Ri7R8{hnm& z*PI8rx{jL9qLWuF?Xxb;iq7WZ{jQUVI?jH%O%wyb(ctT^HTfFz*nJL zkxz$(QLo~(Jv+r%b-_J?cag>LcWsq8=D1KwI%e@*+AEW2hjNjn8X}+nXR>elRzWJJ z{@1C$i6+eg;lF7TBus{|#}Glpm|&VqiagT{8qLwHXHF1aR;DCOI594kpa+xti3Gpz z6vwI*uby;xhvzGv&4RAsuf>WQOydI_aNByGL8I!8zTM~p0huf(^OM*MYX=X|UdPpR zf0A4Od|$L|;-ydopcbzSd6|a#+4qLIT10lSt8Z5r?~PG^_>g+y$ztpT_>KLqLMW)yVY5xw*+Q5Uw19tt~d5$>>6$)&fQ$~>^-a1{4>%w5y(@9y|)a}*r-<3`{bpL zR|CXVv5vVi&%oC7m5_Zx%>6heM53GxyEmQ&Wzd76JEjDYCkr< zAs9uEsT;0>CJ~*BNsKz)#~&jsoKk;mdhz9QKa*A&l;aMBInk4r-s|#ILqB;bM~sgD zualzgQ&P&kq(~nO-K)r}<<9XrGsQ`JX+ClJ%VSHYo|G7?Ft6HH4QdwEkVaEp*$U2C7!2aXQ=YD2#ZdZ)PMIkx*LUA8cG9Qf) z_w3k+-Lad5ZEfwj(|`ZPzy8T&PZ7skd^JilrbkuvjAWSbcT48&eKID>)$V`qaOSux zQVyT)P3yTa2@x+eqavlfnOcq7-rvs(KXYt8?*^#{j#;IX80Q5eZwW@qq}@r{z9DU? z@Ji?Xv*J4XHuo2{V$6!-;9bMK7e-D(;BT-%GB@i#Rx5=u{`;-MyMNyPJOIxo2ns5E zOex`3?&b!2R`ygYF1o;0aTt1ZW12Vok3~pYsIL(O%FUDiBjqx^c$vnoYq$Kf^M!dvR$`~ z{wlIvzK=ZTyhhqm1H?oW-AHZj@0M5uRA{D3_#eYn$i^U}x2KlCU( z$xR)>lnX5#9S!g9+Y*7MQHnh7#fum1=DuK^y~rZ`5NmeeI$Z&&L56hq_bP-2JAPr# zr0o}2<>%+m^<`cJd1DlcfPeteTW@`H-TLSBW3$0Lti;4b&{^SFeck;YMh316SIesPiQjsuTFoQXDC{Gx+ zD$z=6{+WO87XSX5kg%AT9!Rtx{-y5;3szh(gYZ8&{l5EuWk%r%*WIkSr3e~yA(!uC zqsZ?VzpAx|qUzHGmvKq-RD%7-ziK%N>v-e(_3QK2($XQ_n?=J#dvAqta`1jvGGljs`&-q(W2vwgx&=Ca{BDEmJNJ~qrlJ^7)q&t){T1o%C zijdJeW}yG|*d(9${BwSh6$!&VMm+zl4srBx@8!S#_`m=K?!fq-6Y8-~uB$EfR_JXPq`t5+pPy5@V+5E0t1naYGEB_-(!X#~)Eg-Eh0PjaLT zG2FjTfyhRIxWbntk+UEcP}kpoVRNCc9yFQq4caff8BU*bvJeUVwqb>Nw4~JJbol!_m?46Fe?zYU;V6VkTq?GDNFy%kbkDw7o zHNop!;&s197_az=(=srK-Z2}LMG>LAp@$D*XE6K0w|~+6=N^P&q1blyF<;1(Xo??PsNP;+#GEW^%4J0mhidE_r~- z1%*M{O*zK!I*_v3Jpv63VR!fH?!zE+#?XLq?37eis z5GM@G8XOjSH?EEZftuH1QiNW4dAC6}mI3M0kl|dM~!8<4m~nxh*3z1t;`1i@P5BAaYu$v3Cg0$44-c2-N@^p%I*k z50M~%Z2y}IsgJDphAnHtpYADEViCbOX;P^AC6SYp>-h<~e{3c_*KWEu1yLjqlqSuV zlkpk$)@Re@tB3u1LhD%lfUMAAS#7;fS=&EZEHnV5NKKp!4O2nuC>x};&ZE3bOG_U~ zNc7IP+iJ2$1~f2#w$Dg~8QJF2x@&OA6c=~We=NeNaR22u z3{tPD6L_8E)3TB0c;Ao&ea(Rh+3bCw0)C6H+I}%Yni6?Fj5jUd|5qrgCM74Y!+6%Z z`uY>v9tXv(n@;iS_EEBl=V$)2VNgSi;f>DsPtQU8&}Ca`c+jtHac!c$rL*(Y6W$z3 zPBTB~B{}g#LtzsDCDMF>A`g}N(@A*ZOQ=*6r)m;%Se&!&CpgNMT8w_5o<8YmEn#5v zul7Xc4c^kEW-~g4phgRkbr3Zl0u{#(%~COU!l-|NC1TgJR+r-)NNM{ClgQ}+V8Nv5 z7sqw`KY2?{f#jsGu$jU8YI+y-9-XNqI?!9x*w{!*PoJXJB>DkS2mo0?u<@X#mkj;T zD&rC@9!~8*J48*oO@3hgG>MAV8y>tJT;R&O{7d8*p@6 zp3q2Ljv)Otv>c?)7zoD5_3-yB$`hJHuim)v$n@{Y5vIR+)1f$glb8JaZ)`LR^n3$n z3N3$$0AHgx%`eF-D0D-&j`z!#;G>k(B4NuZyrs7&Ue}21n1KE&@L{KymY4HgfB%%< zoN%(}15+XX_J5#1nj{P~hx>wym>Ma!ZE0^85*H6p>>q>`LVa53%S5IPFKqNS7(Zsq z9_>QzvRjWuFSaAaq&IcvXFDq(i02#3r2iao1na zZe;wvkmcCuNsM5_GyE09Zh+U*+skIub@>kTnJBtB#;X@2(OF<0Xdb}+ITOnJ({BU+ zypP~>F&Z~gxTabK8NO2}6_AC5&Q-&%#Ph^`rf{}-!6!Wb^y$I8Xe^>w4-$%C1%jZP z357y{%f^PCEkQUno#n^%)5l8gDiwA­^YIYRx??Zincc?nZv=%6p%+S(d=O9f#- z0Mh(XX!JQruQ?>@?$d~JUom}I)u{+@dn^J68zc@(gU>n5hrVZ*jhOgTB~Ax9CGuzlD*cf_=suTV%e}T`+l6{Ci6r zs0Yxt1e@7@h@w=HY|LqvYF_;S{>@zm>38X+(gswv?ZXKzuX-ACZqmoWAz6{{Ti@Dh z0!I}L=1d4o&e%sMc^jhda6lweC;LgkE2I5ET6tr>H&OORo^B%^NbrVNuzt49ya2Qf zoq&MA{QB~6X>d;3XzHkM%R6jrY*4hrMS=aSDAFKF2B~5&T6c%ENd|&UFf0aqKe%~# zgfbKtHo=!l^}+!4`TVVQ@-iA<`o@nAJW~;p(f4gr@5~_6tN-)|qW>7Gq%+~14ifc) z$IuGn3?5Y&>>3K`TlXq@2fO?YMc{A`TsYcid(4HfwY3$|TK@vD3q+LW_rWneTI4Sa z0=~Gz=J)FVmAJvLY>G@csG)h4^x~*CGuU>gJGzfbwl9G|pf(jvODngu7uj!oYW}otZwgmn~Kw%{X4Sx)**ALM~ z=(g`+N1%?;4GX3KxN0p#{MU1}dXE4E)tvbUzZ*MoD?w`na_t0*H8}tA%B>G5Yr1rP@tx)sVFG}p=le1I0)Qx=iXYp zFuuiESXgL1_k|XMPYk_0ZE_`eMES*Sk1h^Z7@&g7`oE&@@k~HLA;Lb3Aj51 z0XBmYfR%I71%y^BxaRu2O(sP?I;a*@)^Op@P=3)IMhiuV zSxuy~1|){bF#H_P94AD63{<@Fq7_wCZh$8{g_0|^(43c2QdGPKO^G+&#l@XMu^E54 z!V&tGngJev62w50d#g1U5p?)i8qh75^F})P?I)fawQH~tbDJRXQUMl_6eYa9GTI2L zDF8EJSNoUS&Wc}`2jS4%{#lV?F|g8)BqiUvFAU_uT0xiU+^x?Y9be}oBO(}F-P}&0 zpv~|sLbK3%6G7LlE=W+;H8nLEYn-heObxgxGeXT}^#D$~SeAOJG<2aUsA_8~aD$48dKCCoJk5BnIoxzz#$P>8>C zbauv5S67dlAz_dt^6~LGT?!qZ1QXxCU&&saoTQ7JHr|cd<-`kPi?Q5_-KDr^@lI>} z&2T))=WZ?4Z~=2rI%i;8@l4d+z~6fJGPqi)0w+>-(t|8u~-f zj|+0}z;ay^6clU#BZ-7-;8O^r0i5JJuIsWnZx|!bd17XU9>jONLklEQL#$=>%@|H|MkKJy%*>qY z&yH2e*YyFI%wF!%#2GH;G|74M=Hpu`c@NIf@N1kreH!oCvu8S>{|wt(1k@B8pXOFE zbR`IAV-6Cy?ZsgriQ3m}b!E_L!J(m{VY8c)CBAe?P2_Xbtx>Owk0cMsD^=vWe~koC z7tT*8gbikA{fUGOA`=o=!989}m5yT^Dz#(<%E&u7I2fY(i+}@&0L~GEp(6$MkKV!; zEKnTd`BFfwF_=5JWVC<+$bgE$`Zpy4^MAxGRGoar8&2)7^gwp zrw1g-fgz&M2ZLF>zLF}oBr zE-ec@puhP7vzj-cvrlkk;wk5WsEgur;e?q*?Hjz}Js=DO;CN}cG`FYZafS2d0)3;l zsz*~O4Xb$?n3B^2);2bRnM&EW?H3d^s$IWLI|3XHbo%up5j@2jK!V_4{Er_$eyMhI z`3Rt*%wajE)_tluly&J0pf|X* z-sNXNzX@8CJdO_T0S5Df&wbnuM+6B?_V=O6?EOQpJof`iEQs>L;oS zK<%78LrC2KL<94W7a{Zr-wjKHLV+Pf+SHxS2>0ax7Yz?YOi_^(&aeQm?+-1hI`=-m z>bZQ+n<(mz$$1cd!AAi>1Tk()OA89c#l`icH&u$9*D_Q z9j4}-LanW>eS(=thtcU%h#A0=2fi-3;{0#OkW$&+@m| zV1yO~{9r4XK%*J#06K9wOb21*4iC1qt7brL2wd*>|%3 z!NER&Uo|u|P$-BBpio*~18n$Ih9@<2|G*mKe` zj^B;DdS~)1c>j!55U74W?!G5w87m?p@+jiKS(3X&0h@i1c4Sunrf8C42>a z^X-wWSFc?8^j(E1iMxx8@hOcf4fGC&2OXEk=!d!QE#Esw%}WjtQxITPKB&M05&?-R zDrh_}gTNO?B@r+&F)a-hp9Q@&Q9;2M<-StF#T^iqBcKeNCZ%L5s_~*XK*#6EC@Q4I zA2x1VQD|=UuDEz=M?4?p{QUeQ85uE0M=ol&4Y}#*=_k`IVGvhx4j%m-O9jkBFug>? z6^Nr?nhTfp}VD&&UCol7xlwIz#wFWQ1{}nZ^UgK}oNQy?JuOgi4BCWEjNrK5Th=pVT)p(wk}u z25q2+Iy$telF=^!UMz099BdA(ck!%Vxqkf&fT}u}rxO+%8wi?B$QUsoza011n-O6i zP4aWg&=@qOWR;2wW9{)Y9wYiYeg> zvk(eLaaq&B6fl5I);~HZBxCO2f{E(k2D?&^3_{6PF9pPM8c;MjpK}p~7_4|V&^v@E z;1`0r?SYi@f^3cs%*R5;DFAV(v9O$$nu?5gz(OJ7eYQV3p37&PM4dc!N(Y93!5J5U zLSZ=Y95CrWUw1}#Wi5$<~IrP@f#szM4?~{HN$`> zBtt>*EmvE>Z0sWi%4(wSRF-OC>etWhM4dh(Y4Oe&!C~581*N6Uh^iaFCafDdasa{S0IiVg+nGxaMWJ8@#dTnddl@MTe;n+= zM?CIhCuBM{GUCT&HN>j#fIq+iCy1zhrb@+rpP4bR8-?R3aZwM#K?DQEZS{;bpbJB` z_#R#2G`k^{n^+Z+SpH+SBRZMGat{U#bQc=QAV#*kn;1zb6zKbRTRJ}d@sZV}BInFD z7U?*T2*aDlz4)DRJj#~u49cUOEi71&;1)6Dot^kVlL9*_2)OSyjNX`^#Bg@Xx&|14kdhG?^jK1|0WiKd z%$lnMyYArNfQ*7e=77P&4IkV_SPTgG0p=2doj^EyKzld(+?d3ZevcZb;_d;MMgX>> zlN0id%<3f#*ArY{fF(a)`X)NsA2@bwV49voS^x~1T#dUM@7_`&0=jGYVLBdn%~5*# zjgK2W62@QCiQygXckH9LRtLyCs|bq5NiM;tKx9}NxRX;dGBPl6uO6s8$TX1OH6;an zW?)wd9})$Y3DL%cv?v5B$lN)EG5~HuC-1XYu8WEa`AY(-063!$5@pFRC$k|`5Iis1 zH?3h6Y(REVHe-)(XOGuz zj_Bn%3TzNS1cE;aMs7HRqFOTn1cih!5YY-^PhSG6bJA3s?|)wQ0?o6vcGj9FON05w zy{U+AeDd_^Ss(#Gj7w2mDf~w{7zezoVj;rt8~!Q~E@}#xjn}F37?a(tWyC0iFsaONdkLpx&c5xVVeV4++=0%V@)n=0u znBUjhpIq44*@fZ3rR2BivxDLsBY^o22rKV=03qusRK9UH9Wf~>1j3Y*l&~1LHs7?* z==NdIr?=H1($w-eus6{}31%qb`3TY;2mpiF(9l4>bOf8}tg^DQiRRO%PbcQ)ZXAq* zKhDyu?0+&2>6!WckmcoN9|#&y-oO{^0J`~5GB-CTUlno*GGGx}g((t|cu4p=SCtBz zh3w|do2N?=TP>HX#b=@k-_=<0tuL{%?6YUK{T&U%rfl;kDz*Y%$}j z@N|N6IKWE4N>kpGHYx|`1~$9*_h>~~84eP^8N5Vv!nqf}4GrNz;5B~)?1(Nbo%2DF z_O@m9Z&Bw_O$`lX)yBvretH~jd&uvH`xu@}`E9!d-%W-R20Ij#W4pg&ZQ=sFNx#YB zNO>VVNkVF~q8f+GzP`STMKCxM4>bu!{qB&*;r{QQJ+AB5o2PBr_2F8uJ;}MOo<#4O z9Puk>snAZ_ltb+=SR;|^FUMR~`5=NWH0*rcwELZV+-*Cq)=hDm{KG+Ie^5IFr$*|? zP^i|~5)jXx=USPZO!`&cU0@&q$tcy-fli|3&qB+ex%KX>9=smopyIY8LJ0#TOEIW9 z+|4XXxcjvKt*IFxzqh+c&Z@z_IMkA!Dw6sRL#9iYZgzCw1%cQ)EriZ z`zTktrn=1acfpvuICp}-zdtb&^P!Oa(+Ke7Tu9&Gp4IAD6}KMDpXvXtao64*DN>F- zE(j66FoUFq4jqqR6N`^pLY})vBo7jACQc&f84c!hA3GSce6QLXdP+x2`w;SP&dwE4 z_blV>bhZJdd|lKY^A=a#jb*nS1YS`IY zTNA^X(SfmgQSIpRaBykhfA_i zS3pwkdO$#c-EO>qhgL<)Ari!b4?Y_%j<67%oJt>e7yeIsZyt^H_l65UN~VZL(qJem zk(5$0MPy2nRAf#eLMSt34rwry%rhksN*Oa%B4fsknUX1Egmdlso!>d{TJOK-y#Ktd z@A|H#cs$Sk?0w(YeO=dm?`L%dz#B33B0F}-2piWB846XR8(!ZUD`#+kB@bhF=B}hQISeY45_yVlTu@&qowfa zot*OKI+vGr@m$F?bg4v2f zw?;4RQOL9%{<<#eX84bQL~V$l)7f%3Y7Up`oD^PUM800*)YS z<2w}`92`+^SpWjKc)yQMPs`!_u$7Bi)I3i``CyrF8XrJPt^S}gNi%^KgAo15Fy zQ|4ztMuR$}p;qJH{MR3L`3!OE4BaUP&Q$z%`Vk`~+PXOF?Rq{QZKuXdS+6oP*TJpw z)#MpHeK3@6dJ4_n?TWFvgEUg8F;Az9@7i@z*t}JpGT#{+h-Zq(i=x>N3P5IQaE)Sh7#1q z(vy?r?}lP@>K5&>kGrBaP)boDVF2rbt?T?yl2X2|%s-+K+Cor@?!|R{qobp8>(g3= zKZ=i-dSlR-i6qLYwAupPAzJ4E-7X{~3MWtV`rw_yaxk%#!Gm9s)7C2{sG-WvH;mfT zn%;+VGTnNPFBs72K~^<(PV8$H#0&N898|dxId`|aEC2rSEFF(4>2wtKM@D>^mPI}f z#mCLf?Z8OW{ZA^%7~-qY8acD@5<+huyLFU=Gc^U-C4XJhUJL2IQs4Dzg_z+RYGRfc zG;`_rqxRKwbTtY^CQ#-y6@y&C!t>5rbrn$!`EA`H=fD2~)ieE{&TJg0`I{~nk&dAf zk6AMHo!jqWcsRfK%=q|jTS)*p_qP4CG9veA+(4Wb0XFglf9lGY|6JABxRHXfn%{0r zkHJjCc=qUZWM*#QQd3tc4< z=f+V$v2AMt@lHhnxUbLku0r}}ps)!UP=~?P()5K)3%n&O>x@uX7f_IW>hXRm&;a$D znJM4<`nnONX()LMqwnNiKeN9M5F>+FRBWr`IC5x+ccAgq`m<1-5=JLc6L#T@j89FK z;&DLUupey|n&6a~-ML4A%VSDs_mG6j#)MqgyYB8XzHp5l8kU@U6Tj4I9^6uxH8a*3 z`20CPseD{qT#QE3Cy?)k1kUc>P-g{4$r|vKO%?sVL#t5J@Bw)R6*^igGe= z`ymHr0ZLtd(f0UUE4#~=F9*DN@|I-Uy17wt;*xSPWeLq|_v3`Pnp_(#+2PfSjW zO;8gD2+@djUdr_ZR*ZI@tMaT4`ZSuKCzO>xcH6lbQC@0P}Gc&v#ezX!StFu zNleGII)XK38ZVW8uCA5?4$aHYC)1_jgA6ilCyGw(OdUozOJfr;imLsOAG0aCNPNv1xb;t&XxF* zcfG|qeR2^moG8VIk&VcFLWsVNVyT(+E*!gfc{LW6Ft3n_i<%ORk)Co*IH(*DJ$yhS zkis7|qQfx1_S+5E_oDC)5DygL(s^esemWcD4t5FW`7S)+a!9z3LT$`Q1$sOC5PTRUsMPMv$!DEI83|MY1Eg$MgW zLWQrNnwFP^un;nrrY|EDvp6!8De;=t-qA5M)%6^h@h$ydq5?_%jnG{VwVrB2;R}>6 zWjN;v-^0B6b6GcS3i**c(BIE~X|%gr|0Q2?mN2~lg=cAr?-NED0z&gXV-_AEK+c)N zOfbd5XjTwQxHLa3NV%UX_9-rYQ4^}GLj*8cMBm1vG*o55yFlfDcMQu?PEmgk_am4w z6oYjGXK&ujDfUn_DK|I%*;t123ab`{f|;OD4FHHJ;to?qBu3wR6!xVnQzOh+CnX>V^=6MwCf@36T&L^>up@yr$?#lmmf3oKg>UaEU!BmjFvA9&dAXGX<- zgHpPP)n#D=9l)Gw%Q{RcgOgtik{9&O$r#nCbG8>aw+)JdN2y!2K%4ZV_L9F~N+JBP zt|JwFL%QtCi?bYrAx0F7HmVFU`k}Rr?*^k0ZK~fvCUVfd%AE~UA=qqY;~mgYRb7y zyYK|+46R)zVh<3OghIskJ!jO`ogFJdJ=z7Gt~uXPgw%PsRaq)cqZeEh&?GMu_?Ot{ zimuziJRkZu)8+CRhuKj9=Vy6YvztGD`t%;oJ0{9tNB+BN5r=G!Eemp{7x&FQR;!a$ z7@47cMkCzIKcw|4_-OS*9Gb4h+0F{W_l1Z1!;YrdX}k!cF~hv=u=}E-<8h6J+-r%= zLWj-IDhNA2diYRry6T2Gf=1n{*}8l0@?dz0A4*Yd^^R#;wY&4$*(ZEE4ln-sL!^Dk z>#Hy;69Y2yY+QrzmbQ^nD(kweLF|E+=H?>g7`IW+=1*T?ZvK1Ab-;_bZ!=wGs#VLC z^_?XpG7kAl^}72)x8!xID&V#QhG>I&=W_{nOWSJWPOjauWi@y-6zPT#FB}?qjdER8 z&0}(t0yNXgLj<%G8wa|I-PL536zcbDc+4!k;T!Y2>^~#4gal5^OcyU++?>X6@!d@U zP#E5rsY+z7ufK+T-v4P_&L@YXlo0oy{yNUXgf{z zpE@BcyV3I(4;7L@0pib|<=#JKhQ zjiaqO=6eO2+B*$93!IH2@5t7_PEMwKcXNdAtS;CBv0oo{@K$JV?)-2rMg54);Qivm zT-0l}i_uAm@DioBN?SL~g5x~T*^Cs8XO&K;qhZ&5ej!78ew%4U(3OjqE`_&@9g1H( zwUAl|+LKa*D#x<7oXLIII8pJ{>(?hinlsKojJRKG@=kaWf51~*ez^=q!GF7?@Zvip z*n{S3ChFyjXH(T=?;Q`D^*|v|yC)pP-uVnF&1g8$6n#^FPY%E&)putjCI*QpIcUbBVp}=`C6|2Q;*3Sg0U5bt8&uf%yb5 zU{EhS1=>AxsDtCT$gGqiG=a$rQlvaOw~kq3vp@SG2FgoJHFo!X2lCzt0gC7Pt1OaM zQBCIe0YL}v`(%pgEEp(@vO(CbXZ6GKeOcGkjaGT`9)`okU|hNzcq722t+l~Zy~^?I zu|yIozC|yfwio^#FqrBzikv}F71LHZxB0LcTX=Xlr$-U1u2W1C&<1Z=H$Lw4^|wo) z_GMyJIS(FO56^7vFvt4!JH9awd9vaT7LLD$dw98@9yYJLCh79$O_0r2xb2@rwgEs8 zKOC~FqOV_AB6kI6m9s43CR4}Q08XL`T}C=Cj-OOGaa?__aCxEd)3%qZ2*-Uj%9(~j z2XYU&%r;TT2|k}!@zf=djI@jzcZ3P@ASMpx2lAx~4=^CPfPSM;x}e7&cuWz$H5zX7 z;ui++2ERECYZZ2HPeQ1UG~|v*=b4LUxRk*{{Op*_*@_DzJ56Orm2kb(Wg#-D>A|X2 zkXfp`H^8ZKK0Oi}DHx(wa4WXaU%?y@A9c+eD?=+x;dgIihbBSc#Cnw?QtQr$M2dDZGc#CSfp8h%jj`lF$+(H);pgXvS`GmocFNtLIKXK* zp>l)snx4{wq4+h!9BbDSYKp=M3YYtON8TQM7m83UhTM^3$CP>j7n5|BnI-?gK~6=t^felYh&d!EL!jJK;Im_e`#w>4; zrnACRnrJo*q^C&7sQSQ7FgSvilH9r+UAyj8RuDZ>&=;eMULd*8&<7fDXDo(q%O{Ot zUlR)g|A_W{{D6aaesTwmT{-M~YocvHa z1bM=LAWS&O1No7lAh9mQ37?a*2}<5WmvkID1E|#RtE*KgET_e%gZ?o$;H0UfzykdJ zHK=7%#KBz=ql3X zlutS!wNEFd_uJ0S&ZgH)q8l0kn}PDaw)RZ2I$zu{;eiWCL}#RcI(B|k{^uhkRTtGTQ1Jqx-YoGh)zubAl`v2#=ET-x1#LP%rfSp z(CO;xKETx#FPBj)#Ltc+ZzTwg;=|o7?&|7Vr@K<7VE)J6f84h6VpeefY|7CkcG1J8 zpkS-@{29(H8L9$M{_`0HZWl+c@_>UL?Apz-4gpjTFlveU7qBX!HDNM;>!wXtre0OE zRAqdtxu(G$_J*317@iR_dCIO33zO5_OX3`ew~e&xp?4n(q%|qz#pByIZHFkw`PV(g zZq`&PZ@(`NWqdboqIuuis!1Uqb^XAXC|0nWM0&g5VL`|nh@Ya)*{-gw4W`+4NSp~@ zTCkZeCVgvI*w4?eOifGcC+oM#7tchgaAmfp09vTHv3C-zGcvcY1h>!s9~_iH-k|oS znm_10k!OuLB#Ge6N>5KO_u3j6H(~^8&T2@hdp_QP_j;Y+0sL+S(_5C7e`!R}EJW@P z`C3~LEn?lTJk*nn+2lcND>xx-JhLv=ZNtHvihR}|`-`*#K*G8_726HFRzj2PzR=b1 zy+AD&ByB2wbBV_LB2CIiiE)U(;ZNT0{-<&asLbS{lEH5eCzIknlbl)HaKqM)3Ukx) zLg#Y4-!R1^KR+Mpe2x7G#fM~%X-TXzD$?MuFh#>sE`t;{JmtJUhTDT>mr8hzLKtKZ zd3yF!D_+FXZ^ZcZjLRR_4gJQUQDAML5Vk@|;V!Lqfzw4;coc^}_1)+m9epwDI0Vmx zO5vwGy1gERX1r6qbV*w~fx0Wpq~2fW`6$AQiX_%)C_m}9Jt$2^ofF@nRaxG+QD@%? zGs9mFe+lq8vuQ?KH6Y&Z}7-sadCH^+6FI@nJPB@Xb zr%xf7&^U#mgsHof#0}?=A|^w4un(?)$ z>_e3z5;wS@5`ZJcaPD|iGUT=bO#0`qvQAg$6GG8XEa&2mGyGOtL5!FEr`?kB#;TVF zByoU>>|xNff98kNNh#{VLh3Q(*>P-C0}u`{)EsQKK%~a4S>iaMjHm+XBdI*){806fGStp}=QLB`df*#d6bXkaTG z2*Q{Ug&As8+~T+P_Hp3&?)=#f8Fw0{`%zKMaD_2$K0tl)-qvD2!2;u#QOdj}_?`YB z1bU&o$Rd^!M@eMl8W9l@vH8I`;!h`@5%>CFn~~OyL>Gn2ttRdy^P9JCxdjB67(7=H z4-&rbCG-_1===`OZE%KtZorEZ zqY`dAqXUfsuRl1fSY$+SnShzs07q~qkYdqh_$w1&AM!VmmxyVk(=T=zg!fK!HZ(Rh?ADfmMDg){PkK1${{Hbd)(5d~E@Cz>WSG&G3z++Q8x2XZGC z#eq==M1{`fZXj0lhOH$98xhfdI5)@0&;rk`1uM<~&L{rMX2>ZBVKgu^l>+y{S2mkj zNIIQ2$@8Az>3kA|^XfB2`US{lhu+B4aWek!wdR*xIBFRV)20xHJK9T9guXt%=#JP` z)QM@uLS$Tga6N?OwQOv}mfW82__}T^UsY>yt~ZdFuz=3)fXBqx@u~ZcU?EsGZSn&0 zWzS<(g@p}~m3V1D8O4A*yTWGF*SmnZ8_7Q}kV!)Lqpkv%Y&on@_hDXu{u<_ZWRoGJ zJ<8p^0Y0CHP&eY}%8*+n0K&Ii8UXPG&wZI;pt#1hhkGcU-gu3FE%E@dXI43ZsfBBR zw2WrHdWGTZe&l)=;zuIzDc4(?ITwCD3WeMN6Lh*R1 zCJM!Ry9EX^e@)j7MC>Ge4=`H@V-9Z~%+X|i5$S=z-W0OzDq1%tBMjLazvmtMNe%Hi z2{b;V1DAy#_WtmGckn08C%Zw=$EN-p@Paq}@6VGWM>6;GzkmLZ_pAQ* zH~oLmEBpWd?*AM8$!9zTz(y0p3gScX2(iAg9lfy=mL_$h1e@?U4#>;P7r{a8g`_26qriEp zT>_>$QD51r8%*GG5ygmE$7ONexFheV!_*)Z(yTROn;ChFq%KLdE?G&tMh^ymiW?FS zX+oxHQo?H@_;GqrESLYhc}_ocUe|deqWFQiTno5(pmk?mYg&UCupvm%=wg&gTGCND ztWLvcGLn`wlA|kCxoL9y9>= ztH1Z`PW^$)=E%*TY_YMi@sbC?chE!k})$RCt&8k5%R`& zdH2q`)3}%xg^e!~6ekZ4vA)qUFc3y*EenfkYK9KxJ)m}$h3y*lpsv9|;!wadO;lSl zhHQZb8xEMEJuu-wDnYII+qZ4R`-Vn0Pb5_N;eIrjV#!Kzzv_daAZjpypm+5!O2h8^ zgOfHnjze`kDBfAWh)pt*Ygp0#vkFlNh|0Jwo}LIQ(b3U20#6>CE`7+OG_^F>)bV=8 zb7fTn2QcqJ5J0ENeOvfQq`MvdNz=@2yW7Up9Tj76k*?Xdw;Mro!m)D*>_Y$?iR|Gn zJbU-91}(~p8rFAi&JHf#p<5D8bC&4t=q8tYUcS5&{z$X&o(jMPPb64F=&q!j{*>qK zi^2W2=!+4B@0uB`$_@xtAbP+DZu4U1AYjtH?Vxa(-<8x2RU3HO!xjla2VDHVaCylO zU3@THM~2@I8nptjVRsi-}y|MV<}1xykQ_ zP|5~2?fLVm(_aKHy>SCP;5u+%9UR%jM2U?iWY)Ytf532XR_kkT74|$hAUcXeeV=++ zgH&tbYlxCNgoQW2;UEieM;aeNDv&LOsRqTs#}A+c!bpW~NH(Hzf)IZ8y6T6|MGD|x zDb9{1I>CnQ;(c3CrK(@m9!YtFWxEFpcNZ`s9yuK&V<~V%EsX!raJxcvU5Y_N-;an0 zAmRfK&`J2_l?y^z&uVQGvs1mwz&$dX-T!JQ#%h56T|ptPOstw52D0$-<=WiO&)Z*#+z@>@d8W4rP5I(TnSw;8hs0MVFhd42sDdEBD4I z2sj~_(6THIS%(bY0Qb^^>hFl%I46oie>iJU5*5J*iZE3Hxmw#~<5uD{bZ`&_vF8P^HOC?Go0FR&Ck$08vO)5rOcw^5QU~>R23+KA1sNA#3d*?X253WaqST~f1j9U&f zqUC{R;ah?87M%Ptz`y%pVGKZ-C@e^u0zu}UUl*bDT21uG#wR2k$L2*xHcSN`I(?@v zO1K}2QetlvN*34Fzqrg>d#@7ERv(oilqe8sE74RF$f{#kJZN78Ho$7;=S38MB`{fI zE&ytU+Gm=n{)M_IXEXoWv>jZQeqY7%BXSfWzCbE_V>iJ~;4ti&%1;kh=@pXUI4? zD7C`sY(g|VikBf}3X&YQ3Ta}-IGl79Y1zq>1?Wgm88FdEQJ;ey%o3&`a=5T8Xb?JS zV8BhfB2eooE9$}6-s`%&z=0ak6P1J>DseKs2vX6+P*Riu`~ZVZ=}{dLm<4l>97f!0 z2nPTgY1l^EAeea$z7?CL3j&pPdf>QJI3hPm(cj2sJ-HyW( zz=EQm)K4g*Nn13rnx&EMQMB!G3kj_SsF#ECAEWV%@WrI0B6S@)A?9bt0`d8|goPWr z%4$#-V91N+E)&&_PzL?b=7suoZhoFTWSZssf=q_#3pkoD(OFH$Rg-Id6}IPY>`knZ zpi20W+bEFEDF*v1CcMXl1u{$mOv)z6N1%s_kPZAHD(6%8qpbjbn|-jWk|Ko27|>7f z0qBDj6K>tc_0kDPV1OV_itXYKMc5v&0}%clJ|LxF)!O8HhP5O{!}Gwob0;;@&I!ob zDHXHOy2un4NMH|jLP0uk;R_I?FREqh5Q(>-cgrI!O@I&+m>z{5oXupDkqA`9TDT(KP8GDmE#xxSO$#n^ z2i6jXdGtl@14gle`;8nMny3mwK_GIHS{4(uj#&3Il341$Y%x|{5_V+rh@0yp%SU&) z*HZ8Gp^aAE80WU?bn2TIwC66azIM&|=V;DCmd0~US%zfsyqC7(saemo)4w`&ojI+p zIzgenvufqWwHsI6d7yQVdWJdj3D0K=J-z0;!Y`7F2R`Q64@AsH*mvIQ(6v7*0T6V! z-jEJ*ucfW6=cRoW*wZX5EEEcHJfUR9>0-`N*&`qzi(UcH5?@VL^0|!~9pUBm3ZSn~ z(-KivSLXz$L;A!ZobCq#rnn^}@LZ`5_l2GXKoGFTP$r8N)&^4Iq9Dhp7QeS|MIS$X zy3*Lx74B@za-ji(BP=Y;DJaN{T_S~UEUhIb{XcUP)<7kS5P>Pz#Xzr`z~&oH+JHMtc#|r z~lUZ9u@6=YXpHY3l~I z2fonT_~c?vb1cAHugZVx`z)~H$O}rctHO@_o`f-?g7jv4dC?HKMbed&6a~@b%kN9* za>0f1l{X7k(bDb^6(wC(1fV*~#A-atuQOhn0Fz9~36wI11jZY{j*pBC8xZHubvt7c84rDWF zn)`Wb61#Mq#8?mk=sVk=6vut&Q1GO15#k`r`t|PUzcqmUdYfXPgrcG%A@vY$WQS*` zk0!#^LYmN_NLCfhfZpLm*hJaE9&rv(?v8X!5xj$*NRt6$djti&pFG)2DXOTTUH<*W z)e^}NT>9~y>oVGmu%y7epvldrMGl|W((-%waNXmuupfEhb#VM9O}x&{J=)T`1Ta`Cb^wvVuKd#x5u9LaGOJRDH4Xfr5AV~m3P;DHHPk-^E9C`rhU4XOe&cWWS*jVYLEQ!FkyCIxH z9+5+ovQgYpBiwrnuprz?;nb-zM5~r3-%&Ebk?eubNJwmu;4eETzTN8VhQ!zW>MHZn z(rn?iY+du=rx>M2wkJXpX^N^oKD3&Ffg-gdzZG815@Icz21ww6U*ayjO4ssqw6y~e zN$@hX@cXf^G?E$!xfpGtDgbPb5LO1L-|TP&3I0NX(@`90f=jFGz3}pc!0bRL*Gx9N zNCc>n??5lZe(=#VVs<0hO)u>yuY@15S^>mievp}#iJck;0FN7u{l2$;$0GQ55ui|? z#Ee3z*})q9TZA|SGe>4fK+o!MZfI=O2fzgeBzqa8UuEJnq6(TJZ=c{kuC5+bUcVcF z^F4Aw1_Cd?PTu3me1r>ls=t0%z;uTlTwGLW3q?(G?P=jR7BWpQ_tJ}royXzAd7hS* zf$fH^U}}~=k}Y?#ZLRjF{WVjNbpYtES&kI41v1EyLXiG%P*k8?G?+}xKG0n2_>f>$j0)2GA_5m97L^-&45F=m zAO;>t+t>*6?cF$jHZ%J?DwCkrjb?!T2M@|vTD~l2EG4iQW-`xM0BHmDu!e#vuJeb5*(lsiWC-11i;m}wGnB6WelIZX zYy^qmjf|(eF8XPgfM8w{K@(us(mop90v!D~xWU2sG_WaCxnbUy6!!zp9XnPceTa?) zet{9;nNT#}MeBHP{?xLm^{k@L<(-~mOZc}k?F;p1D=Mm{1D z0i8gE5#5w*4m8 zkNWIg=W7>wygujKi2SklH93qc|1??1$;lCiCS=~PP3Pt@H_6nt<(%({_fJDnkbCCl z+COW0yscQw^C9MW_4F7}POTJSVq((NT{+0NXAeEJdc+}Gl2zbp2VFryL#5`aM65@b zzMP{(#}3Bp z3?-VwN#jvc$JLNlrp+d2xWUmNFsr2m-D1hLk7S!)g2&@SrC;RVjEs36 zxACsI0B{hIlGD-}?F#M3tNZ7!SEH*CmC#B!)KXLVE@*1PwLvRY|MObG^xW#ztEq&m zZd@PEu!o6(fCwCOG6Lz=EpL>;78Oq+9;v)aKv-8r>;`(7ET1qQMNq_V@WgkFwRQrK zL4e;6f#}HC^hjH%NE$*&a%$=iRfnm@4o1ZSl3blCW%r8|S53OLRDj#a*6ehB??G>- z%yH$4DI(BAK&!5yNxA7dHd(;-Xx>oyJ znu`U#m5;KmYx@-1KV6E)L^aN#n{j~=rwoqFEodGOPnw!D*6PgLZQeJ8Y-9PeS%~}# zvT7eA_e8c*#KkL7;*MUZWq_I|a|AvM!Hm0o038{-s%M2JL>*>+e!Ms;4~-o2It&xD zB=8PCcnArDtZf8~Y&BZIm zsD#q0Ypxn_^YHTapcF`c@#0Eo=?{NljAPhNlslA7EE_g>Af&<_l2LD0ilONXxh8Eivt*1=iT5oIxQHsPTWovtgUqGPcNfhQRS{xKWDl;z_>+ADG zqyGj>Q>{S0ZMnEiZ1!u`eBIM8t)#q5w$iP`>2WiCnt5ejHLGDv_L|KP)Pd7UQU5ATomkCA^!nv$%!W)30y~f6%-A z=E(&~kUv20-^HHPZz^zV%K5oD_F!gqbzsiDbCq+)q|UbVJN8^gCzMY>fGmiBr)YC5 zM{`*@D0>^t(E)_pd49uI~0`mv|^e+jE}`CZQh3gg-DStM_7m@o1dSba{~i}^X8c``>tbwY(!24RY6*A zvBBcN9++Jev;VboEcZDxd>Sx1(86jB@_n+BrxC;^nDJi-((#(i#LDswUG<+aLEcS!j{LbpviO-+=PnNPg6091g?_UP=A?Bo`oh7%NmVLX7f$Sc3Zf&QqQyG#Z%1+14aBykO# z2BM2J=KuCaY+Ap4UF3jOniAOV5FxGOO*h8ex#2vQ{#6(@&{L>SWm z%aB=E=&18DxNXY#Crg7e`f=i6pQco0R)l*yRnl(eQ=wX~<&my}RUX+c3B?4gE6K+Q zO86_$Rf(<-Y|osmulTMs`G)`V^6z^V=gw`!k*75@G>mXByyk8}^T$u^!UZ~ng?A^; zIgUKcu@QLH-`@`tE4r*NhZdNOLJMr1>ljGMH*CVdapA*>J=`)F zu8t-}Iu%CdE1Y*y9CdhMOxg?Wp1|m}JLJZ%8yXp{f^CNkeFP~Buwe&!(^_SZWdD3s zWu@Pw&b0vrN29X3x`(gRjE#*4Rh2n-1Atl%M!TY6vwEG8SI_Y9>a%ChB4$u`baZG6 zicE8F*tChzU*W|;AQU<#?;ah<%IzQdkh9H%Bir2lcH)6%&ENF{ktei1k13x%ZE>rO z)wT)^vfpRDY5qNLQG2bMm_*Unjm`vfml+VF@Xr411YRSZ3+!9A5O>pzEn3u}R3I)B zPBE`D_|c=1i@S0Y5P>?LJi-pAM6Y!`o}d;J9qpEX^jaVqQa$L`tDZmajm=v9a0OrR zUF~&cA3o55u(q4|x!GlYXji}p!|K(9F7r9TogaC(hDW?;*A=+Xa3(q?dO&BE8T^j$2s?!6hji>81CH=GBSqy`a=_Ivb9iDq3D) z#v$Q!!7%L?VgeM3z*o*dS)-kWU+nW?W<@Gx@ja1Oxjwt!zDf2iZ1}3re%Co&$&HHQ zIKjOZ(m_?qJMz&8XeDR!z_G}SJ$EfsXY3!z?@WIF{JTw;&Kt{OJ*wxn0X8oQ^YV`uf!XP?yrGf+iplGEV94 zT?WivGG*Kij$dl$I*ded>0->GoHB15OFNjEk+Bwoe5Fu6QaE?-_DPEx9v-$omT``6 zqn1h-t-tGo$DCaP>79@;h%30`OV**^D~wGA!R#CvXo->W7XER;x&DHNh8J2FbOy%i zLy%tKLmr1mN)PpaL=K0{efgcBzZxEk)uUhl|NlgU@&^w{|N8ZNaGCsCb3nLod#zC$ zG(j1gwH-5_~`fc z3aov<7G#uM)=Ga^egdi|YrX@Nn~-V4ZrIWp;ZhJ=^LtBE2Dfq zSPBpVWcE4A_5BA9ya(Q$K(~@K_X9Hi=VuOp4QO9iB%Wgrz3g|DYps>LE4=tX!vv25uV!~j{t6iAO z6r;}9q5|o61CsBhcQ;R^XJuVZHf)WAL7W9e1tD!Qp`@5b!uijf5XilkAO=+=0x8? z7afr(cib@)mMC|y^|>yzbqY%y#QEPU>9Q5{8u1ID$9H^g!VvV7eyPt|_IW3;g}15yZxX=K+2&0CTg5YSq}-NThoclRXf5;_DyUe|fjB zE*Lu3E?vwHz#66qjkki8CPFu?bmAa+34#aXxC+%e#wXpz_b$yh6|(bdS7CMl9m;0n z&qUGDyt(Y)mCu7gi`hmXY%mUjC;`oeVNai~&b_q8!eZ=Z)2k-07e<>XWV{(PX6!mW zSOPKbX1D3JYgQ1wf`Wtfu~*W{*@u*`T)6_pi38~dUP2EL4brh43SCUE@)^jz&{aV8)?XbAW5v2P@&5aHY}Hy zoB`%uHs!xHBeTIdp`MfF!z3QuDbbR!3sb!B0IjNPXk@y-tTJgMfn>jk2rCgF!CL5< znGx?SfvuwoS(wM@QB$M(TqmC$Otdt(L?YVF&pSW{?n2O?gR!>7T(|VlN|afh9E732 zzkJX$gBk*vMl?_Uu4V{xD1%!y`^K2U&^DFrgnkL zwz^^|Dk^FKwFIG2(Go0J`&--9$YvPqK-7HF z^{MmqBN8QOr3!HG_wU~?>emK&N(`<6Ppva9_=B4!ZG>Q3iTp=6GkA>1jwSQrbWCaw ziSzRE64?(B2La*vr?K9;*V^OY~%ylq2ybK<*?! z0_P6@E(c6JQNG1O4^86XQqRC{5aJkz$Pyz*u(@Nd#c93Q;;z1&_}_N6Z;X#jREQoe&+63-LSAQZt#F`_s$XX0!?0akc_*L}_2!9nOobbp}z zW3;ckW|~)A+zx~%Y9%G5j^WqPWzaU(1^S-IQGnlg&mYz#li~!SllUuM=;Slt&E#SJ z3^5S2+}@*&bwU$)(5YSs#l%OCt)q{PfZJIbk8$YSB$KxM0|F)>$8duE$CiaUSw_p@ zh{G1qDG(`Wv4wCSPPE>+#_#Xw$QsfPF!K!lQqL38%+i<1A<>1Tas{re5~q1l|d zc+nTz9y-U3kW7FdyaM@O?W(`@57&2Q6WDYBKgm!_LhozArA1tlq-;yAN#l TJb%+m!5;-#l@l+d_3r!+tS1%% diff --git a/docs/source/models/supported-models.md b/docs/source/models/supported-models.md index 0340660acf8f..7b32a6ce9dfb 100644 --- a/docs/source/models/supported-models.md +++ b/docs/source/models/supported-models.md @@ -96,7 +96,7 @@ statuses for the same architecture in the two matrices. [^17]: Kimi K3 has no MTP or EAGLE-3 head, and its DSpark checkpoints are not compatible with plain `DFlash`. [^18]: NGram and standalone Suffix Automaton (SA) use model-free drafting on the PyTorch backend, so they are not listed in individual entries. This does not imply universal end-to-end support: compatibility depends on each model's multi-token verification and cache-management paths and may be untested or explicitly restricted. [^19]: KV cache reuse for hybrid recurrent-attention models requires an explicit recurrent-state snapshot policy, such as `kv_cache_config.mamba_state_config.periodic_snapshot_interval`; the model default disables reuse when no snapshot policy is configured. -[^21]: Supports text, image and video inputs and one-model MTP with 1 to 5 draft tokens. Requires the GLM-specific Transformers revision documented in the deployment guide, whose `Glm5NextProcessor` performs the image and video preprocessing. Beam search and the FP8 KV cache are not supported; disaggregated serving requires the Python NIXL transceiver; KV cache block reuse requires `kv_cache_config.mamba_state_config.periodic_snapshot_interval`. See the [deployment guide](../deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md). +[^21]: Supports text, image and video inputs and one-model MTP with 1 to 5 draft tokens. Requires the GLM-specific Transformers revision documented in the deployment guide, whose `Glm5NextProcessor` performs the image and video preprocessing. Supports FP8 KV cache and attention data parallelism with MTP. Beam search is not supported; disaggregated serving requires the Python NIXL transceiver; KV cache block reuse requires `kv_cache_config.mamba_state_config.periodic_snapshot_interval`. See the [deployment guide](../deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md). # Encoder-Decoder Feature Support Matrix (PyTorch Backend) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py index fb63748c497d..0ad3fd315240 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py @@ -82,7 +82,7 @@ merge_attention_forward_args, ) from ...trtllm import TrtllmAttention, TrtllmAttentionMetadata -from .kernels import kpool_expand, kpool_score, kpool_update +from .kernels import gather_fp8_kv_rows, kpool_expand, kpool_score, kpool_update from .params import INDEX_SENTINEL, GlmKpoolSparseParams @@ -243,6 +243,10 @@ class GlmKpoolSparseAttention(TrtllmAttention): #: lanes; their outputs are discarded by the slice. _KERNEL_HEAD_COUNTS = (64, 128) + # Bound selected-KV staging independently of the configured context length. + # At the published 2112-entry selection this uses 132 MiB of BF16 rows. + _FP8_QUERY_CHUNK_SIZE = 64 + def __init__( self, layer_idx: int, @@ -472,7 +476,15 @@ def append_paged_state( else: table = state.block_tables[request_index] page, offset = paged_slot_indices(table, positions, state.tokens_per_block) - state.latent_pool[page, offset] = latent.to(state.latent_pool.dtype) + if state.latent_pool.dtype == torch.float8_e4m3fn: + quantized = (latent.float() * self.kv_scale_orig_quant).clamp(-448, 448) + # PyTorch advanced indexing does not implement FP8 payloads. Index + # their byte views, preserving the manager's coalesced page strides. + state.latent_pool.view(torch.uint8)[page, offset] = quantized.to( + torch.float8_e4m3fn + ).view(torch.uint8) + else: + state.latent_pool[page, offset] = latent.to(state.latent_pool.dtype) # Only the [k | gate] columns; the pool-key slice is maintained by # update_pool_keys. packed_dim = self.sparse_params.packed_state_dim @@ -493,7 +505,12 @@ def gather_paged_prefix( state.block_tables[request_index], positions, state.tokens_per_block ) packed = state.index_pool[page, offset][..., : self.sparse_params.packed_state_dim] - return state.latent_pool[page, offset], packed + if state.latent_pool.dtype == torch.float8_e4m3fn: + latent = state.latent_pool.view(torch.uint8)[page, offset].view(torch.float8_e4m3fn) + latent = (latent.float() * self.kv_scale_quant_orig).to(torch.bfloat16) + else: + latent = state.latent_pool[page, offset] + return latent, packed def _rows( self, @@ -687,6 +704,21 @@ def _dispatch_sparse_core( output ``[T, H, kv_lora]``; :meth:`_finalize_output` flattens it to the base-contract shape at the ``forward`` boundary. """ + if kv_rows.dtype == torch.float8_e4m3fn: + # Like DeepSeek-V4's BF16 context adapter, dequantize only selected + # rows and reuse FlashMLA. Chunk query rows so prefill never stages + # the full cache or a [max_num_tokens, topk, latent_dim] allocation. + output = torch.empty_like(q_latent) + for start in range(0, q_latent.shape[0], self._FP8_QUERY_CHUNK_SIZE): + stop = min(start + self._FP8_QUERY_CHUNK_SIZE, q_latent.shape[0]) + indices = topk_rows[start:stop] + selected, local_indices = gather_fp8_kv_rows( + kv_rows, indices, self.kv_scale_quant_orig + ) + output[start:stop] = self._dispatch_sparse_core( + q_latent[start:stop], selected, local_indices + ) + return output pad = (-topk_rows.shape[-1]) % self._KERNEL_TOPK_ALIGN if pad: topk_rows = torch.nn.functional.pad(topk_rows, (0, pad), value=INDEX_SENTINEL) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py index bf32fac7b368..2eb0bf20f70d 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py @@ -22,6 +22,67 @@ import triton import triton.language as tl + +@triton.jit +def _gather_fp8_kv_rows_kernel( + rows, + indices, + scale, + output, + local_indices, + row_stride, + index_stride, + column_stride, + NUM_ROWS: tl.constexpr, + DIM: tl.constexpr, + TOPK: tl.constexpr, + BLOCK: tl.constexpr, +): + token = tl.program_id(0) + index = tl.load(indices + (token // TOPK) * index_stride + (token % TOPK) * column_stride) + valid = (index >= 0) & (index < NUM_ROWS) + dim = tl.arange(0, BLOCK) + values = tl.load( + rows + index.to(tl.int64) * row_stride + dim, + mask=valid & (dim < DIM), + other=0.0, + ).to(tl.float32) + values *= tl.load(scale) + tl.store(output + token.to(tl.int64) * DIM + dim, values, mask=dim < DIM) + tl.store(local_indices + token, tl.where(valid, token, -1)) + + +def gather_fp8_kv_rows( + rows: torch.Tensor, indices: torch.Tensor, scale: torch.Tensor +) -> tuple[torch.Tensor, torch.Tensor]: + """Gather selected FP8 rows as BF16 and remap sparse indices in one pass. + + ``rows`` is ``[N, 1, D]`` with contiguous features, ``indices`` is + ``[queries, topk]`` and ``scale`` is a device FP32 scalar. Invalid indices + remain -1; no unselected cache payload is read or dequantized. + """ + selected = torch.empty( + (indices.numel(), 1, rows.shape[-1]), dtype=torch.bfloat16, device=rows.device + ) + local_indices = torch.empty(indices.shape, dtype=torch.int32, device=indices.device) + if indices.numel(): + _gather_fp8_kv_rows_kernel[(indices.numel(),)]( + rows, + indices, + scale, + selected, + local_indices, + rows.stride(0), + indices.stride(0), + indices.stride(1), + NUM_ROWS=rows.shape[0], + DIM=rows.shape[-1], + TOPK=indices.shape[1], + BLOCK=triton.next_power_of_2(rows.shape[-1]), + ) + return selected, local_indices + + _FP32_MIN = torch.finfo(torch.float32).min diff --git a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/kimi_k3_kda/kda_mtp_decode.py b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/kimi_k3_kda/kda_mtp_decode.py index dee10d230030..e00a1ecd670e 100644 --- a/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/kimi_k3_kda/kda_mtp_decode.py +++ b/tensorrt_llm/_torch/cute_dsl_kernels/blackwell/kimi_k3_kda/kda_mtp_decode.py @@ -136,6 +136,10 @@ def kda_decode_mtp_kernel( bos = cu_seqlens[i_n] eos = cu_seqlens[i_n + 1] slot = ssm_state_indices[i_n] + # V2 can interleave many layers between state slots. Promote the index + # before multiplying by static strides: the element offset can exceed + # INT32_MAX even when each individual stride fits in int32. + slot = Int64(slot) h0_idx = slot * HV + i_hv hk_off = i_h * K hv_off = i_hv * V diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next.py b/tensorrt_llm/_torch/models/modeling_glm5_next.py index ffb87dd717ac..6986e62d1ff2 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next.py @@ -401,14 +401,15 @@ def mixed_kwargs(self, layer_idx: int) -> dict[str, Any]: The attention module splits the packed tokens at ``num_ctx_tokens`` and runs its context rows through the prefill kernels and its - generation rows through the decode kernels; everything outside + generation rows through the decode or verify kernels; everything outside attention runs once over the whole batch (the vLLM layout). The KDA layers do this split inside the shared mixer from the metadata. """ return { "num_ctx_tokens": self.num_ctx_tokens, "prefill": self.sparse_kwargs(layer_idx, "prefill"), - "decode": self.sparse_kwargs(layer_idx, "decode"), + "generation": self.sparse_kwargs(layer_idx, self.gen_phase), + "generation_phase": self.gen_phase, } def sparse_kwargs(self, layer_idx: int, phase: str) -> dict[str, Any]: @@ -603,11 +604,6 @@ def mamba_metadata_cls(self) -> type[Mamba2Metadata]: def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: text_config = get_glm5_next_text_config(model_config.pretrained_config) - if model_config.mapping.enable_attention_dp and model_config.spec_config is not None: - raise ValueError( - "glm5_next does not support attention DP with speculative decoding; " - "set enable_attention_dp=False when enabling MTP." - ) # tie_word_embeddings is false on this checkpoint, so lm_head is a real # weight rather than a view of the embedding; it is asserted, not assumed. if bool(getattr(text_config, "tie_word_embeddings", False)): @@ -1702,12 +1698,15 @@ def forward_mixed( *, num_ctx_tokens: int, prefill: dict[str, Any], - decode: dict[str, Any], + generation: dict[str, Any], + generation_phase: str = "decode", ) -> torch.Tensor: - """Context rows through the prefill path, generation rows through the - fused decode path, one TP reduction over the concatenated output.""" + """Split attention by phase and reduce the concatenated output once.""" + if generation_phase not in ("decode", "verify"): + raise ValueError(f"Unsupported generation phase: {generation_phase}") ctx = self.forward_prefill(hidden_states[:num_ctx_tokens], reduce=False, **prefill) - gen = self.forward_decode(hidden_states[num_ctx_tokens:], reduce=False, **decode) + generation_forward = getattr(self, f"forward_{generation_phase}") + gen = generation_forward(hidden_states[num_ctx_tokens:], reduce=False, **generation) out = torch.cat([ctx, gen], dim=0) return out if self.tp_all_reduce is None else self.tp_all_reduce(out) @@ -1717,6 +1716,7 @@ def forward_verify( kv_lens: torch.Tensor, metadata: AttentionMetadata, tokens_per_request: int, + reduce: bool = True, ) -> torch.Tensor: """Generation phase with ``tokens_per_request`` tokens per request. @@ -1739,7 +1739,7 @@ def forward_verify( """ tokens_per_request = int(tokens_per_request) if tokens_per_request <= 1: - return self.forward_decode(hidden_states, kv_lens, metadata) + return self.forward_decode(hidden_states, kv_lens, metadata, reduce=reduce) num_tokens = hidden_states.shape[0] batch = num_tokens // tokens_per_request if batch * tokens_per_request != num_tokens or kv_lens.shape[0] != batch: @@ -1773,6 +1773,7 @@ def forward_verify( metadata, AttentionInputType.generation_only, rows_per_request=tokens_per_request, + reduce=reduce, ) @@ -2116,8 +2117,8 @@ def forward( """Runtime entry: derive this layer's cache arguments from metadata. The executor packs context requests first, then generation requests; - the two phases run through :meth:`forward_direct` separately, which is - exact because every non-attention operation here is token-local. + attention handles the two phases separately, while the remaining + operations run once over the packed batch to preserve DP collectives. ``position_ids`` is unused: the text path is fully NoPE and the indexer derives its positions from the cached lengths. """ @@ -2134,36 +2135,15 @@ def forward( ) derive = runtime_ctx.sparse_kwargs if runtime_ctx.num_contexts > 0 and runtime_ctx.num_generations > 0: - if runtime_ctx.gen_phase == "decode": - # Mixed context+generation batch: one pass over all tokens - # (hyper-connections, norms, MoE, the o_proj reduction), with - # the attention module splitting the rows internally between - # its prefill and decode kernels -- the vLLM layout. A second - # small-batch pass per layer would cost a whole decode step - # per mixed iteration. - return self.forward_direct( - hidden_states, - phase="mixed", - all_rank_num_tokens=all_rank_num_tokens, - **runtime_ctx.mixed_kwargs(self.layer_idx), - ) - # Speculative verification rows run the replay verify kernel; they - # keep the split path (prefill rows, then verify rows). - parts = [ - self.forward_direct( - hidden_states[: runtime_ctx.num_ctx_tokens], - phase="prefill", - all_rank_num_tokens=all_rank_num_tokens, - **derive(self.layer_idx, "prefill"), - ), - self.forward_direct( - hidden_states[runtime_ctx.num_ctx_tokens :], - phase="verify", - all_rank_num_tokens=all_rank_num_tokens, - **derive(self.layer_idx, "verify"), - ), - ] - return torch.cat(parts, dim=0) + # Only attention is phase-specific. MoE must see the complete + # packed batch exactly once: DP ranks can have different phase + # mixes, but must issue the same collective sequence and counts. + return self.forward_direct( + hidden_states, + phase="mixed", + all_rank_num_tokens=all_rank_num_tokens, + **runtime_ctx.mixed_kwargs(self.layer_idx), + ) if runtime_ctx.num_contexts > 0: return self.forward_direct( hidden_states, diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index c6763fdb8d67..35ad04c9e5b4 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -3009,10 +3009,6 @@ def _create_kv_cache_manager( # Glm5NextCacheManager. Must come before the is_mla(...) route: the # glm5_next text config carries MLA fields, but only 11 of its 45 # layers are sparse MLA. - if kv_cache_dtype == tensorrt_llm.bindings.DataType.FP8: - raise ValueError( - "glm5_next does not support FP8 KV cache; use " - "kv_cache_config.dtype='auto' with a BF16 latent cache.") if max_beam_width > 1: raise ValueError("glm5_next + beam search is not supported yet.") if not estimating_kv_cache and kv_connector_manager is not None: diff --git a/tests/integration/defs/accuracy/references/gsm8k.yaml b/tests/integration/defs/accuracy/references/gsm8k.yaml index 747bc9177fe5..e17f87bceb7e 100644 --- a/tests/integration/defs/accuracy/references/gsm8k.yaml +++ b/tests/integration/defs/accuracy/references/gsm8k.yaml @@ -549,3 +549,11 @@ zai-org/GLM-5.3-Flash: - quant_algo: FP8_BLOCK_SCALES spec_dec_algo: MTP accuracy: 92.91 + # FP8 KV uses the same regression gates as BF16 KV. + - quant_algo: FP8_BLOCK_SCALES + kv_cache_quant_algo: FP8 + accuracy: 92.57 + - quant_algo: FP8_BLOCK_SCALES + kv_cache_quant_algo: FP8 + spec_dec_algo: MTP + accuracy: 92.91 diff --git a/tests/integration/defs/accuracy/test_glm53_flash.py b/tests/integration/defs/accuracy/test_glm53_flash.py index a0937185882e..7be614093059 100644 --- a/tests/integration/defs/accuracy/test_glm53_flash.py +++ b/tests/integration/defs/accuracy/test_glm53_flash.py @@ -282,6 +282,39 @@ def test_mtp(self, tp_size, ep_size): task.evaluate(llm) assert_acceptance_length_for_llm("TestGLM53FlashFP8::test_mtp", llm) + @skip_pre_blackwell + @pytest.mark.skip_less_mpi_world_size(4) + @pytest.mark.parametrize( + "attention_dp,draft_len,cache_dtype", + [(True, 3, "auto"), (False, 0, "fp8"), (True, 3, "fp8")], + ids=["adp-mtp3", "tp-fp8-kv", "adp-mtp3-fp8-kv"], + ) + def test_attention_dp_mtp_and_fp8_kv(self, attention_dp, draft_len, cache_dtype): + """Manual regressions for the supported attention-DP, MTP and KV-cache combinations.""" + kwargs = self._llm_kwargs(4, 4) + kwargs.update( + enable_attention_dp=attention_dp, + enable_chunked_prefill=True, + max_num_tokens=4096 if attention_dp else 16384, + kv_cache_config=KvCacheConfig( + dtype=cache_dtype, free_gpu_memory_fraction=0.5, enable_block_reuse=False + ), + ) + if draft_len: + kwargs.update( + speculative_config=MTPDecodingConfig(max_draft_len=draft_len), + max_stats_len=-1, + enable_iter_perf_stats=True, + ) + with LLM(self.MODEL_PATH, **kwargs) as llm: + self._assert_glm5_next_stack(llm) + assert llm.args.enable_attention_dp == attention_dp + if cache_dtype == "fp8": + assert llm.args.quant_config.kv_cache_quant_algo == QuantAlgo.FP8 + GSM8K(self.MODEL_NAME).evaluate(llm) + if draft_len: + assert_acceptance_length_for_llm("TestGLM53FlashFP8::test_mtp", llm) + @skip_pre_blackwell @pytest.mark.timeout(900) @pytest.mark.skip_less_mpi_world_size(4) diff --git a/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py b/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py index 732d22f6a45e..8e7aac05ca10 100644 --- a/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py +++ b/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py @@ -22,9 +22,17 @@ kpool_score, kpool_update, ) +from tensorrt_llm._utils import get_sm_version pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +try: + from tensorrt_llm.flash_mla import flash_mla_sparse_fwd + + HAS_FLASH_MLA = flash_mla_sparse_fwd is not None +except ImportError: + HAS_FLASH_MLA = False + HD = 128 KPOOL = 4 TPB = 8 @@ -265,3 +273,89 @@ def reference(): tables.copy_(tables.flip(0)) graph.replay() assert torch.equal(output, reference()) + + +@pytest.mark.skipif(not HAS_FLASH_MLA, reason="FlashMLA not available") +@pytest.mark.skipif(get_sm_version() < 90, reason="FlashMLA requires SM90 (Hopper) or later") +@pytest.mark.parametrize("heads", [16, 64]) +def test_fp8_sparse_core_matches_dequantized_cache_and_replays_graph(heads): + """Exercise the selected-row core used by production prefill, decode and verify.""" + from tensorrt_llm._torch.attention.backends.sparse.glm_kpool import ( + GlmKpoolSparseAttention, + latent_pool_rows, + ) + + backend = object.__new__(GlmKpoolSparseAttention) + backend.kv_lora_rank = 512 + backend.softmax_scale = 256**-0.5 + backend.kv_scale_quant_orig = torch.tensor([0.5], device="cuda") + backend._FP8_QUERY_CHUNK_SIZE = 2 # Exercise multiple chunks and a short final chunk. + generator = torch.Generator(device="cuda").manual_seed(71) + storage = torch.randn(12, 4, 512, device="cuda", generator=generator).to(torch.float8_e4m3fn) + pool = storage[1::3] # Nonzero offset and coalesced page stride. + rows, base, stride = latent_pool_rows(pool) + q = torch.randn(5, heads, 512, device="cuda", generator=generator).to(torch.bfloat16) + positions = torch.randint(0, 16, (5, 67), device="cuda", generator=generator) + indices = (base + positions // 4 * stride + positions % 4).int() + indices[:, -2] = -1 + indices[:, -1] = rows.shape[0] # Preserve FlashMLA's out-of-range sentinel semantics. + + def reference(): + decoded = (rows.float() * backend.kv_scale_quant_orig).to(q.dtype) + return backend._dispatch_sparse_core(q, decoded, indices) + + expected = reference() + actual = backend._dispatch_sparse_core(q, rows, indices) + torch.testing.assert_close(actual, expected, atol=2e-2, rtol=2e-2) + stream = torch.cuda.Stream() + stream.wait_stream(torch.cuda.current_stream()) + with torch.cuda.stream(stream): + backend._dispatch_sparse_core(q, rows, indices) + torch.cuda.current_stream().wait_stream(stream) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + captured = backend._dispatch_sparse_core(q, rows, indices) + storage.copy_((storage.float() * 0.5).to(storage.dtype)) + graph.replay() + torch.testing.assert_close(captured, reference(), atol=2e-2, rtol=2e-2) + + +@pytest.mark.parametrize("context", [False, True]) +def test_fp8_cache_storage_helpers_preserve_coalesced_pages(context): + """Check cache-write and compatibility prefix-gather helpers. + + The production model supplies topk_rows and does not call gather_paged_prefix; + its FP8 attention core is covered by the sparse-core test above. + """ + from types import SimpleNamespace + from unittest.mock import Mock + + from tensorrt_llm._torch.attention.backends.sparse.glm_kpool import GlmKpoolSparseAttention + + backend = object.__new__(GlmKpoolSparseAttention) + backend.kv_scale_orig_quant = torch.tensor([2.0], device="cuda") + backend.kv_scale_quant_orig = torch.tensor([0.5], device="cuda") + backend.sparse_params = SimpleNamespace(packed_state_dim=4) + storage = torch.zeros(9, 4, 8, device="cuda", dtype=torch.float8_e4m3fn) + latent_pool = storage[1::3] + index_pool = torch.zeros(9, 4, 6, device="cuda", dtype=torch.bfloat16)[2::3] + state = SimpleNamespace( + latent_pool=latent_pool, + index_pool=index_pool, + tokens_per_block=4, + num_contexts=1 if context else 0, + block_tables=torch.tensor([[2, 0]], device="cuda"), + ) + backend._cache_state = Mock(return_value=state) + latent = torch.tensor([[0.0, 0.125, -0.4, 1.1, 5.0, -5.0, 500.0, -500.0]], device="cuda") + packed = torch.tensor([[1.0, 2.0, 3.0, 4.0]], device="cuda") + positions = torch.tensor([0] if context else [[0]], device="cuda") + backend.append_paged_state( + latent, packed, positions, object(), request_index=0 if context else None + ) + decoded, actual_packed = backend.gather_paged_prefix(1, object(), request_index=0) + expected = (latent * 2).clamp(-448, 448).to(torch.float8_e4m3fn).float() * 0.5 + torch.testing.assert_close(decoded.float(), expected) + torch.testing.assert_close(actual_packed.float(), packed) + assert torch.count_nonzero(storage.float()[[0, 2, 3, 5, 6, 8]]) == 0 + assert torch.count_nonzero(index_pool[:, :, 4:]) == 0 diff --git a/tests/unittest/_torch/modeling/test_glm5_next_contracts.py b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py index c71fad89dd76..24dacea071e8 100644 --- a/tests/unittest/_torch/modeling/test_glm5_next_contracts.py +++ b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py @@ -2,8 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 """GLM configuration, loading and vision ownership regressions without checkpoint files.""" -from types import SimpleNamespace -from unittest.mock import patch +from types import MethodType, SimpleNamespace +from unittest.mock import Mock, create_autospec, patch import pytest import torch @@ -16,9 +16,12 @@ audit_glm5_next_checkpoint, ) from tensorrt_llm._torch.models.modeling_glm5_next import ( - Glm5NextForCausalLM, + SPARSE_MLP, + Glm5NextDecoderLayer, Glm5NextLinearAttention, + Glm5NextRuntimeContext, Glm5NextSparseAttention, + glm5_next_tp_reduces, ) from tensorrt_llm._torch.models.modeling_glm5_next_vision import Glm5NextVisionModelBase from tensorrt_llm._torch.pyexecutor.config_utils import get_glm5_next_layer_masks @@ -143,7 +146,7 @@ def test_cache_manager_routing_guards(route, monkeypatch): @pytest.mark.cpu_only @pytest.mark.parametrize("fp8_kv_cache", [False, True], ids=["bf16-kv", "fp8-kv"]) -def test_fp8_kv_cache_rejected_before_manager_construction(fp8_kv_cache): +def test_kv_cache_dtype_reaches_manager_construction(fp8_kv_cache): from tensorrt_llm._torch.attention.backends.sparse.glm_kpool import Glm5NextCacheManager from tensorrt_llm._torch.pyexecutor._util import _create_kv_cache_manager from tensorrt_llm.bindings import DataType @@ -163,9 +166,7 @@ class AllocationReached(Exception): pass with patch.object(Glm5NextCacheManager, "__new__", side_effect=AllocationReached) as allocate: - expected = ValueError if fp8_kv_cache else AllocationReached - message = "glm5_next does not support FP8 KV cache" if fp8_kv_cache else None - with pytest.raises(expected, match=message): + with pytest.raises(AllocationReached): _create_kv_cache_manager( model_engine=None, kv_cache_manager_cls=Glm5NextCacheManager, @@ -185,11 +186,10 @@ class AllocationReached(Exception): dtype=torch.bfloat16, is_draft=False, ) - if fp8_kv_cache: - allocate.assert_not_called() - else: - allocate.assert_called_once() - assert allocate.call_args.kwargs["dtype"] == DataType.BF16 + allocate.assert_called_once() + assert allocate.call_args.kwargs["dtype"] == ( + DataType.FP8 if fp8_kv_cache else DataType.BF16 + ) @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") @@ -213,23 +213,6 @@ def test_encoder_only_factory_materializes_attention_weights(deferred): assert projection.weight is not None -@pytest.mark.cpu_only -def test_attention_dp_speculation_rejected_before_model_construction(): - config = SimpleNamespace( - pretrained_config=_config(), - mapping=Mapping(world_size=4, tp_size=4, enable_attention_dp=True), - spec_config=object(), - ) - with ( - patch( - "tensorrt_llm._torch.models.modeling_glm5_next.Glm5NextModel", - side_effect=AssertionError("must reject before constructing layers"), - ), - pytest.raises(ValueError, match="does not support attention DP"), - ): - Glm5NextForCausalLM(config) - - @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") def test_kda_shards_preserve_fp32_gate_parameters(): config = _config().text_config @@ -380,3 +363,133 @@ def run(lengths, discard_prefix=False): broken, _, _ = run([5, 8], discard_prefix=True) broken_l2 = (broken.float() - reference.float()).norm() / reference.float().norm() assert broken_l2 > 0.03 + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("tokens_per_request", [1, 4], ids=["decode", "verify"]) +def test_attention_dp_mixed_batches_keep_one_full_batch_moe_call(tokens_per_request: int) -> None: + # Rank-local phase mixes differ under ADP. Exercise the real decoder + # dispatch and FFN wrapper with lightweight math stubs. + batches = [(3, 1), (3, 0), (0, 1), (2, 1)] + counts = [ + context_tokens + generations * tokens_per_request for context_tokens, generations in batches + ] + for rank, (context_tokens, generations) in enumerate(batches): + mapping = Mapping( + world_size=4, tp_size=4, moe_ep_size=4, rank=rank, enable_attention_dp=True + ) + contexts = int(context_tokens > 0) + metadata = SimpleNamespace(all_rank_num_tokens=counts) + context = Glm5NextRuntimeContext( + manager=object(), + num_contexts=contexts, + num_ctx_tokens=context_tokens, + num_generations=generations, + ctx_cu_seqlens=[0, context_tokens] if contexts else [0], + cached_lens=[0] * contexts + [1] * generations, + state_indices=torch.arange(contexts + generations), + kv_lens=torch.tensor( + ([context_tokens] if contexts else []) + + ([1 + tokens_per_request] if generations else []) + ), + metadata=metadata, + gen_tokens_per_request=tokens_per_request, + ) + hidden = torch.arange(counts[rank] * 4 * 8, dtype=torch.float32).view(-1, 4, 8) + reduction = Mock(side_effect=lambda value: value) + attention = SimpleNamespace( + tp_all_reduce=reduction if glm5_next_tp_reduces(mapping) else None + ) + for phase in ("prefill", "decode", "verify"): + implementation = MethodType( + getattr(Glm5NextSparseAttention, f"forward_{phase}"), attention + ) + setattr( + attention, + f"forward_{phase}", + create_autospec(implementation, side_effect=lambda x, *args, **kwargs: x), + ) + attention.forward_mixed = MethodType(Glm5NextSparseAttention.forward_mixed, attention) + connection = SimpleNamespace( + pre_mapping=lambda streams: (None, None, streams.mean(dim=1)), + post_mapping=lambda output, residual, post, comb: residual + output.unsqueeze(1), + ) + layer = SimpleNamespace( + attention_type="deepseek_sparse_attention", + layer_idx=1, + mlp_type=SPARSE_MLP, + self_attn=attention, + hc_attn=connection, + hc_ffn=connection, + input_layernorm=torch.nn.Identity(), + post_attention_layernorm=torch.nn.Identity(), + mlp=Mock(side_effect=lambda x, all_rank_num_tokens: x), + ) + layer.forward_direct = MethodType(Glm5NextDecoderLayer.forward_direct, layer) + layer.run_mlp = MethodType(Glm5NextDecoderLayer.run_mlp, layer) + result = Glm5NextDecoderLayer.forward(layer, hidden_states=hidden, runtime_ctx=context) + assert result.shape == hidden.shape + reduction.assert_not_called() + layer.mlp.assert_called_once() + mlp_tokens, all_rank_num_tokens = layer.mlp.call_args.args + assert mlp_tokens.shape[0] == counts[rank] + assert all_rank_num_tokens is counts + if contexts: + attention.forward_prefill.assert_called_once() + torch.testing.assert_close( + attention.forward_prefill.call_args.args[0], hidden.mean(dim=1)[:context_tokens] + ) + else: + attention.forward_prefill.assert_not_called() + generation = getattr(attention, f"forward_{context.gen_phase}") + if generations: + generation.assert_called_once() + assert generation.call_args.kwargs["metadata"] is metadata + torch.testing.assert_close( + generation.call_args.args[0], hidden.mean(dim=1)[context_tokens:] + ) + if contexts: + assert generation.call_args.kwargs["reduce"] is False + if tokens_per_request > 1: + assert generation.call_args.kwargs["tokens_per_request"] == tokens_per_request + else: + generation.assert_not_called() + other = attention.forward_decode if tokens_per_request > 1 else attention.forward_verify + other.assert_not_called() + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("generation_phase", ["decode", "verify"]) +@pytest.mark.parametrize("reduce_output", [False, True], ids=["attention-dp", "tp"]) +def test_mixed_attention_splits_only_attention_and_reduces_once(generation_phase, reduce_output): + gen_tokens = 4 if generation_phase == "verify" else 1 + hidden = torch.randn(3 + gen_tokens, 8) + ctx = torch.zeros(3, 8) + gen = torch.ones(gen_tokens, 8) + reduction = Mock(side_effect=lambda value: value + 10) if reduce_output else None + attention = SimpleNamespace( + forward_prefill=Mock(return_value=ctx), + forward_decode=Mock(return_value=gen), + forward_verify=Mock(return_value=gen), + tp_all_reduce=reduction, + ) + result = Glm5NextSparseAttention.forward_mixed( + attention, + hidden, + num_ctx_tokens=3, + prefill={"marker": "context"}, + generation={"marker": "generation"}, + generation_phase=generation_phase, + ) + expected = torch.cat([ctx, gen]) + (10 if reduce_output else 0) + torch.testing.assert_close(result, expected) + attention.forward_prefill.assert_called_once() + assert attention.forward_prefill.call_args.kwargs["reduce"] is False + selected = getattr(attention, "forward_" + generation_phase) + selected.assert_called_once() + assert selected.call_args.kwargs["reduce"] is False + torch.testing.assert_close(selected.call_args.args[0], hidden[3:]) + other = attention.forward_decode if generation_phase == "verify" else attention.forward_verify + other.assert_not_called() + if reduction is not None: + reduction.assert_called_once() diff --git a/tests/unittest/_torch/modules/kimi_kda/test_kimi_kda_fused_verify_parity.py b/tests/unittest/_torch/modules/kimi_kda/test_kimi_kda_fused_verify_parity.py index 273395f9b7a8..7c4078286f36 100644 --- a/tests/unittest/_torch/modules/kimi_kda/test_kimi_kda_fused_verify_parity.py +++ b/tests/unittest/_torch/modules/kimi_kda/test_kimi_kda_fused_verify_parity.py @@ -78,7 +78,9 @@ def _is_blackwell(): @torch.no_grad() -def _make_runtime(seed, aux_stream=None, checkpoint_fp8=False, *, num_heads=H, use_full_rank_gate=True): +def _make_runtime( + seed, aux_stream=None, checkpoint_fp8=False, *, num_heads=H, use_full_rank_gate=True +): # A real KimiLinearConfig (not a SimpleNamespace) so the runtime sees the # same config surface it does in production. ``linear_attn_config`` carries # the per-layer KDA params the runtime reads plus the (unused here) @@ -201,23 +203,36 @@ def _rep(name, a, b): @torch.no_grad() @pytest.mark.parametrize( - ("num_heads", "use_full_rank_gate", "checkpoint_fp8"), - [(H, True, False), (H, True, True), (16, False, False), (64, False, False)], - ids=["full-rank-bf16", "full-rank-fp8", "low-rank-h16", "low-rank-h64"], + ("num_heads", "use_full_rank_gate", "checkpoint_fp8", "large_state_stride"), + [ + (H, True, False, False), + (H, True, True, False), + (16, False, False, False), + (64, False, False, False), + (64, False, False, True), + ], + ids=["full-rank-bf16", "full-rank-fp8", "low-rank-h16", "low-rank-h64", "large-state-stride"], ) -def test_fused_vs_sequential_two_rounds(num_heads, use_full_rank_gate, checkpoint_fp8): +def test_fused_vs_sequential_two_rounds( + num_heads, use_full_rank_gate, checkpoint_fp8, large_state_stride +): from tensorrt_llm._torch.modules.multi_stream_utils import with_multi_stream torch.manual_seed(0) B = 4 T = M + 1 rt_seq = _make_runtime( - seed=1, checkpoint_fp8=checkpoint_fp8, num_heads=num_heads, + seed=1, + checkpoint_fp8=checkpoint_fp8, + num_heads=num_heads, use_full_rank_gate=use_full_rank_gate, ) rt_fused = _make_runtime( - seed=1, aux_stream=torch.cuda.Stream(), checkpoint_fp8=checkpoint_fp8, - num_heads=num_heads, use_full_rank_gate=use_full_rank_gate, + seed=1, + aux_stream=torch.cuda.Stream(), + checkpoint_fp8=checkpoint_fp8, + num_heads=num_heads, + use_full_rank_gate=use_full_rank_gate, ) rt_fused.finalize_decode_weights() if checkpoint_fp8: @@ -236,7 +251,19 @@ def test_fused_vs_sequential_two_rounds(num_heads, use_full_rank_gate, checkpoin conv_pool_seq, ssm_pool_seq = _make_pools(B, seed=2, num_heads=num_heads) conv_pool_fused = conv_pool_seq.clone() - ssm_pool_fused = ssm_pool_seq.clone() + if large_state_stride: + # Only four small states are populated. The gaps reproduce V2's + # coalesced layer layout, with the last slot beyond INT32_MAX elements. + state_stride = ((2**31 // (B - 1)) // (K * K) + 1) * K * K + storage_size = (B - 1) * state_stride + num_heads * K * K + free_bytes, _ = torch.cuda.mem_get_info() + if free_bytes < storage_size * ssm_pool_seq.element_size() + 2**30: + pytest.skip("large-stride regression needs 9 GiB of free GPU memory") + storage = torch.empty(storage_size, dtype=ssm_pool_seq.dtype, device="cuda") + ssm_pool_fused = storage.as_strided(ssm_pool_seq.shape, (state_stride, K * K, K, 1)) + ssm_pool_fused.copy_(ssm_pool_seq) + else: + ssm_pool_fused = ssm_pool_seq.clone() cache_seq = _make_seq_layer_cache(B, num_heads=num_heads) cache_fused = _make_fused_layer_cache(B, conv_pool_fused, num_heads=num_heads) From bab6f0693c93bc789fb600141fac17c6cf100075 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Tue, 15 Sep 2026 20:34:30 -0700 Subject: [PATCH 04/35] [None][docs] Clarify GLM KV cache auto dtype behavior Signed-off-by: Ruocheng Jia --- .../deployment-guide-for-glm-5.3-flash-on-trtllm.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md index b6cb19f915e4..5daa1682c203 100644 --- a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md @@ -145,7 +145,7 @@ kv_cache_config: EOF ``` -To use FP8 KV cache with any of these aggregated-serving configurations, add `dtype: fp8` under `kv_cache_config`. This halves latent KV storage; indexer and KDA state precision are unchanged. Selected KV rows are dequantized in bounded query chunks before attention. For large prefills, repeated staging and attention calls can increase time to first token and reduce prefill throughput, in addition to decode overhead. Leave `dtype: auto` (BF16) when prioritizing latency or prefill throughput. The performance curves below use BF16 KV cache. +To use FP8 KV cache with any of these aggregated-serving configurations, add `dtype: fp8` under `kv_cache_config`. This halves latent KV storage; indexer and KDA state precision are unchanged. Selected KV rows are dequantized in bounded query chunks before attention. For large prefills, repeated staging and attention calls can increase time to first token and reduce prefill throughput, in addition to decode overhead. Use BF16 KV when prioritizing latency or prefill throughput. `dtype: auto` inherits the checkpoint's KV-cache quantization metadata, so BF16 also requires that metadata not to enable FP8 KV quantization. The performance curves below use BF16 KV cache. ### Launch the TensorRT LLM Server From 55d955057c4d783a94631fca1b29e3f57f89be00 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Wed, 16 Sep 2026 02:35:08 -0700 Subject: [PATCH 05/35] [None][fix] Honor all stop tokens in greedy sampling Check every single-token stop sequence so GLM tool calls stop at the observation boundary. Add focused sampler regression coverage and document validated tool calling, structured output, and long-context serving. Signed-off-by: Ruocheng Jia --- ...yment-guide-for-glm-5.3-flash-on-trtllm.md | 11 ++++++- .../_torch/sampler/test_torch_sampler.py | 33 +++++++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md index 5daa1682c203..342a31a6875e 100644 --- a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md @@ -20,6 +20,7 @@ The following features have been tested on B200 with four GPUs per worker. Disag * Disaggregated serving with MTP * KV cache block reuse (with periodic Mamba state snapshots, see [`kv_cache_config`](#kv_cache_config)) * Image and video inputs (see [Multimodal Inputs](#multimodal-inputs)) +* Automatic tool calling and JSON structured output (see [Tool Calling and Structured Output](#tool-calling-and-structured-output)) ### Limitations @@ -211,7 +212,7 @@ These options provide control over TensorRT LLM's behavior and are set within th * **Options**: * `enable_block_reuse`: Enables prefix reuse. Requires `mamba_state_config.periodic_snapshot_interval` (for example `256`); without this policy, reuse is disabled. * `free_gpu_memory_fraction`: Fraction of free GPU memory reserved for caches after loading the model. **Recommendation**: `0.5`; reduce it if you encounter OOM errors. - * `dtype`: Attention KV-cache data type. **Default**: `auto` (BF16). Set to `fp8` to reduce latent KV storage; this also works with MTP and attention data parallelism. + * `dtype`: Attention KV-cache data type. **Default**: `auto`, which follows the checkpoint's KV-cache quantization metadata. Set to `fp8` to reduce latent KV storage; this also works with MTP and attention data parallelism. #### `cuda_graph_config` @@ -298,6 +299,14 @@ For `/v1/chat/completions`, pass reasoning controls through `chat_template_kwarg Send images and videos to `/v1/chat/completions` using the OpenAI-compatible `image_url` and `video_url` content types. Video decoding requires `opencv-python-headless` in the container. +### Tool Calling and Structured Output + +For tool calling, add `--tool_parser glm47 --reasoning_parser deepseek-r1` to the server command and use `tool_choice: "auto"` (streaming/non-streaming and MTP all supported). Limitations: `tool_choice: "required"` and named-function choice are not supported, and `parallel_tool_calls` should be omitted (the model can still return multiple tool calls under `auto`). + +### Long Context + +The checkpoint supports a total sequence length of 1,048,576 tokens. Set `--max_seq_len 1048576` and keep chunked prefill enabled; input + output must fit within this limit. On 4×B200 (TP4/EP4, BF16 KV), a validated config is `--max_batch_size 16`, `--max_num_tokens 8192`, CUDA graph batch limit 16, and `kv_cache_config.free_gpu_memory_fraction: 0.6`. This was smoke-tested up to the exact limit (≈1.04M-token multi-needle retrieval and a full 1,048,576-token run) — validating execution and basic retrieval, not comprehensive long-context quality. + ### Troubleshooting Tips * **CUDA OOM errors:** Reduce `--max_batch_size`, `--max_num_tokens`, or `kv_cache_config.free_gpu_memory_fraction`. Keep `cuda_graph_config.max_batch_size` consistent with the server batch limit. diff --git a/tests/unittest/_torch/sampler/test_torch_sampler.py b/tests/unittest/_torch/sampler/test_torch_sampler.py index afa3e8e8359a..b50135821553 100644 --- a/tests/unittest/_torch/sampler/test_torch_sampler.py +++ b/tests/unittest/_torch/sampler/test_torch_sampler.py @@ -928,6 +928,39 @@ class TestFinishReasons: END_ID = FinishReason.END_ID LENGTH = FinishReason.LENGTH + @pytest.mark.cpu_only + @pytest.mark.parametrize("new_token", [5, 7, 9, 11]) + def test_single_step_greedy_checks_all_stop_tokens(self, new_token): + sampler = object.__new__(TorchSampler) + sampler.max_seq_len = 20 + sampler._track_pending_steps = False + request = LlmRequest( + request_id=0, + seq_slot=0, + input_tokens=[2, 0], + max_new_tokens=10, + end_id=2, + stop_words_list=[[5], [7], [9]], + sampling_config=SamplingConfig(), + is_streaming=False, + ) + state = SampleStateTorch( + requests=[request], + device=None, + host=SampleStateTensorsHostTorch( + new_tokens=torch.tensor([new_token], dtype=torch.int32), + finish_reasons=None, + first_finish_reasons=None, + single_step_greedy=True, + ), + ) + + sampler.update_requests(state) + + assert request.is_finished == (new_token in (5, 7, 9)) + assert not request.is_finished_due_to_length + assert request.get_tokens(0) == [2, 0, new_token] + def test_single_step_greedy_updates_finish_reasons_and_filters_completed_requests(self): sampler = object.__new__(TorchSampler) sampler.max_seq_len = 20 From 86fcd937276981d5f14e277c0c670104cc7da624 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Wed, 16 Sep 2026 03:35:43 -0700 Subject: [PATCH 06/35] [None][test] Include greedy stop-token regression in GPU CI Match the existing sampler tests by removing the cpu_only marker. The CPU test list does not include the sampler directory, while GPU stages exclude cpu_only cases, so the regression now runs through the existing sampler entries without adding CI stages. Signed-off-by: Ruocheng Jia --- tests/unittest/_torch/sampler/test_torch_sampler.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/unittest/_torch/sampler/test_torch_sampler.py b/tests/unittest/_torch/sampler/test_torch_sampler.py index b50135821553..62c73ab6699e 100644 --- a/tests/unittest/_torch/sampler/test_torch_sampler.py +++ b/tests/unittest/_torch/sampler/test_torch_sampler.py @@ -928,7 +928,6 @@ class TestFinishReasons: END_ID = FinishReason.END_ID LENGTH = FinishReason.LENGTH - @pytest.mark.cpu_only @pytest.mark.parametrize("new_token", [5, 7, 9, 11]) def test_single_step_greedy_checks_all_stop_tokens(self, new_token): sampler = object.__new__(TorchSampler) From bc4f9d7288e5b106b891a15afd59785c5379be23 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Wed, 16 Sep 2026 05:23:55 -0700 Subject: [PATCH 07/35] [None][refactor] Simplify GLM-5.3-Flash runtime and sparse backend Use prepared GLM page tables and live TRTLLM KV lengths throughout sparse attention. Remove unused prefix/index paths, metadata reconstruction, redundant state fields and forwarding helpers. Reuse mixed-phase attention in MTP to reduce the complete TP output once. Keep cache layout, precision and CUDA graph contracts documented while removing historical bring-up notes. Share PD worker configuration and update contract/kernel tests to exercise the production row-selection path. Signed-off-by: Ruocheng Jia --- .../backends/sparse/glm_kpool/__init__.py | 9 +- .../backends/sparse/glm_kpool/backend.py | 527 ++++----------- .../sparse/glm_kpool/cache_manager.py | 124 +--- .../backends/sparse/glm_kpool/params.py | 9 +- .../checkpoints/hf/glm5_next_weight_mapper.py | 64 +- .../_torch/models/modeling_glm5_next.py | 619 ++++-------------- .../models/modeling_glm5_next_vision.py | 119 +--- tensorrt_llm/_torch/pyexecutor/_util.py | 5 +- .../accuracy/test_disaggregated_serving.py | 19 +- .../sparse/glm_kpool/test_glm_kpool.py | 91 ++- .../sparse/glm_kpool/test_kernels.py | 19 +- .../modeling/test_glm5_next_contracts.py | 103 ++- 12 files changed, 458 insertions(+), 1250 deletions(-) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/__init__.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/__init__.py index 0dc30a53395f..21ae44a85268 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/__init__.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/__init__.py @@ -14,13 +14,7 @@ # limitations under the License. """glm_kpool: the GLM-5.3-Flash pool-compressed sparse-MLA backend.""" -from .backend import ( - INDEX_SENTINEL, - GlmKpoolSparseAttention, - latent_pool_rows, - paged_slot_indices, - positions_to_pool_rows, -) +from .backend import INDEX_SENTINEL, GlmKpoolSparseAttention, latent_pool_rows, paged_slot_indices from .cache_manager import Glm5NextCacheManager, Glm5NextMamba2Metadata from .params import GlmKpoolBackendForwardArgs, GlmKpoolSparseParams @@ -33,5 +27,4 @@ "GlmKpoolSparseParams", "latent_pool_rows", "paged_slot_indices", - "positions_to_pool_rows", ] diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py index 0ad3fd315240..36e78e85dfba 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py @@ -12,59 +12,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""GLM-5.3-Flash k-pool sparse MLA: the fully-NoPE branch of the TRTLLM family. - -``glm5_next`` sparse layers are fully NoPE MLA (``qk_rope_head_dim == 0``, -``qk_nope_head_dim == 256``, ``kv_lora_rank == 512``) whose visible set is -selected by a *pool-compressed* indexer: size-``index_kpool`` pools of keys are -scored, the top ``index_topk / index_kpool`` pools expand back into member -positions, and the incomplete tail is always appended, padded with ``-1`` to a -fixed logical width. Stock DeepSeek-V3.2 DSA selects individual keys and -assumes the 576-wide rope'd DeepSeek geometry (its SM100 absorption route -applies RoPE through ``mla_rope_append_paged_kv_assign_q``, and the shared -``MLA`` module's DSA hook requires the fused ``[q_a|kv_a|k_pe]`` projection and -``qk_rope_head_dim > 0``), so it cannot express this contract without inventing -a fake rotary width. - -:class:`GlmKpoolSparseAttention` is therefore a narrow **subclass of -:class:`~..trtllm.TrtllmAttention`** -- a sibling of -:class:`~.dsa.backend.DSATrtllmAttention` inside the TRTLLM sparse family -- -rather than a fork outside it: - -* **Family** -- it inherits the TRTLLM backend's identity wholesale: - ``support_mla()`` is true, ``is_mla_enable`` is true through a fully-NoPE - :class:`~..interface.MLAParams`, and ``Metadata`` is - :class:`~..trtllm.TrtllmAttentionMetadata`, the typed metadata class the - engine constructs from ``attn_backend.Metadata``. -* **Selection** -- ``get_attention_backend("TRTLLM", sparse_params)`` resolves - ``SparseParams(algorithm="glm_kpool")`` to this class next to the DSA branch - in the sparse registry, and the standard ``create_attention(...)`` dispatch - constructs it. -* **Contract** -- :meth:`forward` keeps the exact - ``AttentionBackend.forward(q, k, v, metadata, forward_args)`` signature: - options merge through :func:`~..interface.merge_attention_forward_args` - (which rejects unknown or mixed arguments), the model layer's pool-expanded - selection arrives through the typed - ``AttentionForwardArgs.sparse_backend_args.topk_indices`` carrier, and the - phase is declared through ``forward_args.attention_input_type``. -* **Cache ownership** -- every paged read and write is derived from the - prepared ``metadata``: the one hybrid ``KVCacheManagerV2`` reached through - ``metadata.kv_cache_manager`` owns the latent pages and the packed indexer - state, and the per-request block tables / visible lengths come from the - ``prepare()``-refreshed ``mamba_metadata.glm_*`` buffers. Callers never hand - raw pool tensors to this backend. -* **Execution** -- the absorbed sparse-MLA core dispatches - ``tensorrt_llm.flash_mla.flash_mla_sparse_fwd``, the FlashMLA sparse kernel - of the DSA stack (``sparse/dsa/module.py``), which natively supports - ``d_qk == d_v == 512`` and 64 query heads (``HEAD_DIM_512`` / - ``Fwd_Sm100_Head64_Impl`` in FlashMLA's ``sparse_fwd.h``) with the same - ``-1``/out-of-range invalid-index contract this model's indexer emits. - -The model layer (``modeling_glm5_next.Glm5NextSparseAttention``) keeps the -module math -- projections, norms, the pool indexer's scoring/selection -(model-layer sparse prediction, as in MiniMax-M3), query absorption, and the -output projection -- and drives this backend only through the standard -contract entry points. +"""GLM-5.3-Flash pool-compressed sparse MLA in the TRTLLM backend family. + +The model projects queries and selects pools; this backend owns paged latent +and indexer state, pool-key updates, and FlashMLA attention. Selection arrives +as global latent-cache row IDs in GlmKpoolBackendForwardArgs.topk_rows. + +Queries have no rotary component and are absorbed into the 512-wide latent +space. FlashMLA consumes BF16 rows; FP8 cache rows are gathered and dequantized +in bounded query chunks. Prepared GLM page tables and live TRTLLM lengths +provide the same cache contract for prefill, decode and verification. """ from __future__ import annotations @@ -87,11 +44,7 @@ def _flash_mla_sparse_fwd() -> Callable[..., tuple[torch.Tensor, torch.Tensor, torch.Tensor]]: - """The production sparse-MLA kernel entry point (lazy import). - - Resolved at call time so a test can intercept the module attribute to - prove the backend is the one dispatching it. - """ + """Resolve the FlashMLA sparse kernel lazily.""" try: from tensorrt_llm.flash_mla import flash_mla_sparse_fwd except ImportError as exc: # pragma: no cover - wheel always bundles it @@ -107,26 +60,12 @@ def paged_slot_indices( positions: torch.Tensor, tokens_per_block: int, ) -> tuple[torch.Tensor, torch.Tensor]: - """``(page, within_page)`` indices for ``positions`` in a paged pool. - - ``block_table`` is ``[..., max_pages]`` and ``positions`` is broadcastable - against its leading dimensions. The pair addresses a pool shaped - ``[num_pages, tokens_per_block, ...]``. - - Returning the pair rather than one flat ``page * tokens_per_block + offset`` - index is required, not stylistic. ``KVCacheManagerV2`` coalesces buffers - whose per-page size differs from K/V's into a shared pool, so the - ``Role.INDEX_KEY`` view it hands back is **strided**: its page stride is the - slot stride, not its own payload size. Measured on this model the indexer - buffer is ``[16, 64, 1, 256]`` with page stride 32768 against a 16384-element - payload. A flat index is only correct for a densely packed pool, and - flattening that view is not merely wrong -- ``.view`` raises and ``.reshape`` - silently *copies*, so every cache write would land in a temporary and be - lost. The pair is also what the accessor's own contract prescribes. - - Callers must mask invalid positions themselves -- this deliberately does not - invent a fallback page, because a silently wrong page is exactly the - cross-request leak the hybrid cache has to rule out. + """Return (page, within_page) indices for positions in a paged pool. + + block_table is [..., max_pages]; positions matches its leading dimensions. + Use this pair to index [slots, tokens_per_block, ...] pools: V2 coalesced page + strides may include other buffers, so flattening would copy or misaddress + storage. Callers must supply valid positions. """ page = torch.div(positions, tokens_per_block, rounding_mode="floor") offset = positions - page * tokens_per_block @@ -134,26 +73,14 @@ def paged_slot_indices( def latent_pool_rows(latent_pool: torch.Tensor) -> tuple[torch.Tensor, int, int]: - """Row-space view of the latent pool's storage for the sparse kernel. - - ``latent_pool`` is the slot-major ``[slots, tokens_per_block, dim]`` view - from ``Glm5NextCacheManager.get_latent_state_buffer``. V2 coalesces - several buffers into one pool, so the slot stride is larger than the - payload and the pool cannot be flattened with ``view`` (it would raise) - or ``reshape`` (it would silently copy every step). The kernel, however, - only needs *uniformly strided rows*: it reads ``kv_ptr + idx * stride``. - Every latent row already sits at a multiple of ``dim`` elements inside - the storage, so the whole storage is reinterpreted as ``[N, 1, dim]`` - rows and positions are translated to row ids instead. - - Returns ``(rows, base_row, rows_per_slot)`` where the row id of cache - position ``(slot, t)`` is ``base_row + slot * rows_per_slot + t``. Rows - belonging to other coalesced buffers are addressable but never indexed: - only ids produced by that formula (or ``-1`` sentinels) reach the kernel. - Pure metadata work -- no device kernel -- so it is CUDA-graph safe. + """View coalesced latent storage as uniformly strided [N, 1, dim] rows. + + Input is [slots, tokens_per_block, dim]. Returns (rows, base_row, rows_per_slot), + with row_id = base_row + slot * rows_per_slot + within_page. Only valid latent + row IDs or -1 sentinels may reach FlashMLA; storage also contains other buffers. + No allocation or device work is needed, so the view is CUDA-graph safe. """ - slots, tokens_per_block, dim = latent_pool.shape - del slots, tokens_per_block + dim = latent_pool.shape[-1] if latent_pool.stride(2) != 1 or latent_pool.stride(1) != dim: raise ValueError( "glm_kpool latent pool rows must be contiguous within a page; got " @@ -172,39 +99,12 @@ def latent_pool_rows(latent_pool: torch.Tensor) -> tuple[torch.Tensor, int, int] return rows, offset // dim, slot_stride // dim -def positions_to_pool_rows( - positions: torch.Tensor, - block_table: torch.Tensor, - tokens_per_block: int, - base_row: int, - rows_per_slot: int, -) -> torch.Tensor: - """Request-local cache positions -> global row ids for the sparse kernel. - - ``positions`` is int32 ``[..., width]`` with :data:`INDEX_SENTINEL` in - invalid slots; ``block_table`` holds base slot ids per request. Sentinels - are preserved as ``-1`` (the kernel's own invalid marker) rather than - clamped -- a clamped sentinel would address a real row. Fixed shapes, - gathers, and ``where`` only, so a captured decode graph replays this - against prepare()-refreshed tables. - """ - safe = positions.clamp(min=0).long() - page = torch.div(safe, tokens_per_block, rounding_mode="floor") - slot = torch.gather(block_table, -1, page) - rows = base_row + slot * rows_per_slot + (safe - page * tokens_per_block) - return torch.where(positions >= 0, rows, positions.long()).to(torch.int32) - - @dataclass(frozen=True) class _GlmKpoolCacheState: - """One layer's cache state, derived from prepared attention metadata. - - ``latent_pool``/``index_pool`` are the slot-major ``[slots, - tokens_per_block, dim]`` views over the hybrid manager's coalesced pools; - ``block_tables``/``kv_lens`` cover the whole batch in executor order - (contexts first). Everything here is a view or a host int -- deriving it - launches no kernel, so it is safe inside a CUDA-graph capture as long as - the underlying buffers are the persistent ``prepare()``-refreshed ones. + """Layer-local pool views and prepared tables/lengths in context-first order. + + Pool views are [slots, tokens_per_block, dim]. Device tables and lengths retain + their prepared addresses for CUDA graph replay. """ latent_pool: torch.Tensor @@ -216,14 +116,7 @@ class _GlmKpoolCacheState: class GlmKpoolSparseAttention(TrtllmAttention): - """Fully-NoPE k-pool sparse-MLA branch of the TRTLLM attention family. - - See the module docstring for the family/contract rationale. Constructed - under the standard ``create_attention(...)`` dispatch when - ``SparseParams(algorithm="glm_kpool")`` is configured on the TRTLLM - backend slot; the sparse registry resolves it next to - ``DSATrtllmAttention``. - """ + """NoPE k-pool MLA backend selected by SparseParams(algorithm="glm_kpool").""" Metadata = TrtllmAttentionMetadata @@ -232,15 +125,8 @@ class GlmKpoolSparseAttention(TrtllmAttention): #: semantics-free by the kernel's own contract. _KERNEL_TOPK_ALIGN = 64 - #: Query-head counts the FlashMLA sparse kernel instantiates - #: (``Fwd_Sm100_Head64_Impl``/``Head128`` in ``sparse_fwd``; any other - #: ``h_q`` raises ``Unsupported h_q``). Under tensor parallelism the local - #: head count (16 of 64 at TP4) is below the smallest instantiation, so - #: :meth:`_dispatch_sparse_core` zero-pads the query-head axis up to the - #: next instantiated count and slices the output back -- the in-tree DSA - #: precedent (``sparse/dsa/module.py`` pads its TP-local heads the same - #: way). Attention is per-head, so zero query lanes cannot perturb real - #: lanes; their outputs are discarded by the slice. + # FlashMLA instantiates 64/128 query heads. Pad TP-local heads to the next + # size and discard the extra outputs; heads attend independently. _KERNEL_HEAD_COUNTS = (64, 128) # Bound selected-KV staging independently of the configured context length. @@ -339,16 +225,10 @@ def support_fused_qkv(cls) -> bool: # -- metadata-derived cache state ---------------------------------------- def _cache_state(self, metadata) -> _GlmKpoolCacheState: - """Derive this layer's cache state from the prepared metadata. - - The single source of cache truth for every entry point below. The - production path is the persistent one: ``prepare()`` with the - ``Glm5NextCacheManager`` attached refreshes the ``mamba_metadata``'s - ``glm_block_tables``/``glm_kv_lens`` device buffers, so a captured - decode graph replays against fresh values at stable addresses. - Harness metadata whose manager does not attach those buffers falls - back to an eager host-side derivation, which is refused under CUDA - graphs exactly like the model's runtime-context builder used to. + """Read slot-major pool views, prepared GLM tables and live TRTLLM lengths. + + kv_lens_cuda includes overlap corrections and MTP rewinds. Constructing this + state uses views only, with no allocation, H2D copy or host synchronization. """ if metadata is None: raise ValueError( @@ -382,60 +262,19 @@ def _cache_state(self, metadata) -> _GlmKpoolCacheState: num_contexts = int(metadata.num_contexts) tables = getattr(mamba_metadata, "glm_block_tables", None) - if tables is not None: - # Persistent path: slices of prepare()-refreshed buffers. No - # allocation, no H2D, no host sync -- CUDA-graph safe. Visible - # lengths prefer the metadata's device-corrected kv_lens_cuda - # (overlap scheduler + speculative decoding rewinds it in-graph; - # see modeling_glm5_next.glm5_next_visible_lens). - # Kept in the metadata's own integer width: every consumer here - # (comparisons, the fused kernels) accepts int32 or int64, and a - # per-call cast would launch one kernel per entry point per layer. - live = getattr(metadata, "kv_lens_cuda", None) - kv_lens = live[:batch] if live is not None else mamba_metadata.glm_kv_lens[:batch] - return _GlmKpoolCacheState( - latent_pool=latent, - index_pool=index, - block_tables=tables[:batch], - kv_lens=kv_lens, - tokens_per_block=tokens_per_block, - num_contexts=num_contexts, - ) - - # Legacy eager derivation for harness managers that do not attach the - # GLM buffers. It allocates and copies, so it must never run inside a - # captured region. - if getattr(metadata, "is_cuda_graph", False): + if tables is None: raise RuntimeError( - "glm_kpool CUDA-graph execution requires the persistent " - "prepare()-refreshed glm_block_tables/glm_kv_lens buffers; the " - "attached mamba_metadata has no glm_block_tables" + "glm_kpool requires prepared glm_block_tables; call metadata.prepare() " + "with Glm5NextMamba2Metadata before eager execution or CUDA graph capture" ) - kv_params = getattr(metadata, "kv_cache_params", None) - if kv_params is None or kv_params.num_cached_tokens_per_seq is None: - raise ValueError("glm_kpool requires kv_cache_params.num_cached_tokens_per_seq") - lens = [int(n) for n in metadata.seq_lens[:batch]] - cached = [int(n) for n in kv_params.num_cached_tokens_per_seq[:batch]] - device = latent.device - # Raw base-slot IDs, NOT get_batch_cache_indices: the latent/index - # views are slot-major, and V2's standard accessor scales page ids for - # its own flattened per-layer views. - pages = manager.get_batch_slot_tables(list(metadata.request_ids)[:batch]) - max_pages = max((len(p) for p in pages), default=1) or 1 - block_tables = torch.zeros(batch, max_pages, dtype=torch.long, device=device) - for row, page_ids in enumerate(pages): - if page_ids: - block_tables[row, : len(page_ids)] = torch.as_tensor( - page_ids, dtype=torch.long, device=device - ) - kv_lens = torch.as_tensor( - [c + n for c, n in zip(cached, lens)], dtype=torch.long, device=device - ) + kv_lens = getattr(metadata, "kv_lens_cuda", None) + if kv_lens is None: + raise ValueError("glm_kpool requires prepared metadata.kv_lens_cuda") return _GlmKpoolCacheState( latent_pool=latent, index_pool=index, - block_tables=block_tables, - kv_lens=kv_lens, + block_tables=tables[:batch], + kv_lens=kv_lens[:batch], tokens_per_block=tokens_per_block, num_contexts=num_contexts, ) @@ -449,19 +288,13 @@ def append_paged_state( positions: torch.Tensor, metadata, *, - request_index: int | None = None, request_ids: torch.Tensor | None = None, ) -> None: - """Write new tokens' latent and packed indexer state to the pools. - - ``positions`` carries the tokens' cache positions (schedule, owned by - the model layer); which pools and tables they land in is derived from - ``metadata``. ``request_index`` selects one context request's table - row; ``request_ids`` (``[tokens]`` int32, executor request index per - packed context token) addresses all context requests at once; - ``None`` for both addresses the generation rows, with ``positions`` - shaped ``[num_generations, 1]``. Callers pass only positions they own - -- see :func:`paged_slot_indices` for why no fallback page is invented. + """Write latent rows and packed [k | gate] state into the paged cache. + + For packed context, positions and request_ids are [tokens], identifying each + row's request. Otherwise positions is [generation_requests, tokens_per_request] + and uses the generation slice of the prepared block tables. """ state = self._cache_state(metadata) if request_ids is not None: @@ -471,10 +304,7 @@ def append_paged_state( offset = positions - page_idx * state.tokens_per_block page = state.block_tables[request_ids.long(), page_idx] else: - if request_index is None: - table = state.block_tables[state.num_contexts :] - else: - table = state.block_tables[request_index] + table = state.block_tables[state.num_contexts :] page, offset = paged_slot_indices(table, positions, state.tokens_per_block) if state.latent_pool.dtype == torch.float8_e4m3fn: quantized = (latent.float() * self.kv_scale_orig_quant).clamp(-448, 448) @@ -490,58 +320,23 @@ def append_paged_state( packed_dim = self.sparse_params.packed_state_dim state.index_pool[page, offset, :packed_dim] = packed.to(state.index_pool.dtype) - def gather_paged_prefix( - self, - length: int, - metadata, - *, - request_index: int, - ) -> tuple[torch.Tensor, torch.Tensor]: - """One context request's cached latent and packed-indexer prefix (host - loop path: prefill only, never captured).""" - state = self._cache_state(metadata) - positions = torch.arange(length, device=state.latent_pool.device) - page, offset = paged_slot_indices( - state.block_tables[request_index], positions, state.tokens_per_block - ) - packed = state.index_pool[page, offset][..., : self.sparse_params.packed_state_dim] - if state.latent_pool.dtype == torch.float8_e4m3fn: - latent = state.latent_pool.view(torch.uint8)[page, offset].view(torch.float8_e4m3fn) - latent = (latent.float() * self.kv_scale_quant_orig).to(torch.bfloat16) - else: - latent = state.latent_pool[page, offset] - return latent, packed - def _rows( self, state: _GlmKpoolCacheState, - request_index: int | None, - num_rows: int, rows_per_request: int = 1, request_ids: torch.Tensor | None = None, ) -> tuple[torch.Tensor, torch.Tensor | None]: - """``(block_tables, kv_lens)`` for the rows the fast-path kernels see. - - ``None`` addresses the generation rows: one row per generation request - (visible length from the metadata), or ``rows_per_request`` rows per - request -- a speculative verification pass -- whose block tables are - repeated per row and whose per-row visible lengths the caller - supplies. An ``int`` addresses one context request: its single block - table is broadcast (stride 0) over the request's ``num_rows`` query - tokens, whose per-row visible lengths the caller supplies (each query - sees its own prefix). ``request_ids`` addresses the packed rows of all - context requests: the kernels index the batch's block tables through - it (row ``i`` -> table ``request_ids[i]``), so nothing is gathered. + """Select generation tables, or all tables for packed context request IDs. + + Verification repeats generation tables per token; callers supply each + query's visible length for verification and packed context rows. """ if request_ids is not None: return state.block_tables, None - if request_index is None: - gen = slice(state.num_contexts, None) - if rows_per_request > 1: - return state.block_tables[gen].repeat_interleave(rows_per_request, dim=0), None - return state.block_tables[gen], state.kv_lens[gen] - table = state.block_tables[request_index].unsqueeze(0).expand(num_rows, -1) - return table, None + gen = slice(state.num_contexts, None) + if rows_per_request > 1: + return state.block_tables[gen].repeat_interleave(rows_per_request, dim=0), None + return state.block_tables[gen], state.kv_lens[gen] def update_pool_keys( self, @@ -549,21 +344,16 @@ def update_pool_keys( ape: torch.Tensor, metadata, *, - request_index: int | None = None, request_ids: torch.Tensor | None = None, ) -> None: - """Refresh the pool containing each of ``positions``. - - Generation rows (``request_index=None``): ``positions`` is - ``[num_generations]``, the cache position each request just wrote. - One context request: ``positions`` are that request's pool-final - positions written this chunk (one per pool, so no two programs write - the same pool). ``ape`` is the indexer's ``[kpool, head_dim]`` - compress APE. All context requests at once: ``request_ids`` maps each - position to its request. One fused kernel, in place, CUDA-graph safe. + """Refresh the pool containing each position using compression APE. + + positions is [rows]; ape is [kpool, head_dim]. Generation has one position per + request. For packed context, request_ids maps each row to a request and only + pool-final positions are supplied, avoiding concurrent writes to one pool. """ state = self._cache_state(metadata) - tables, _ = self._rows(state, request_index, positions.shape[0], request_ids=request_ids) + tables, _ = self._rows(state, request_ids=request_ids) kpool_update( state.index_pool, tables, @@ -583,26 +373,19 @@ def score_pools( *, q_scale: float, w_scale: float, - request_index: int | None = None, kv_lens: torch.Tensor | None = None, rows_per_request: int = 1, request_ids: torch.Tensor | None = None, ) -> torch.Tensor: - """Fused pool scoring -> ``[N, P_cap]`` fp32. - - ``q`` is ``[N, n_heads, head_dim]``, ``weights`` ``[N, n_heads]``. - Reads the cached pool keys directly (work proportional to each row's - visible length, not to the buffer capacity); pools that are not yet - complete hold the fp32 minimum. Generation rows by default; for one - context request pass ``request_index`` and the per-query-token - visible lengths ``kv_lens`` (``position + 1``); for a speculative - verification pass over the generation rows pass ``rows_per_request`` - (tokens per request) and the per-row ``kv_lens``. + """Score complete pools into FP32 [rows, pool_capacity]. + + q is [rows, heads, head_dim], weights is [rows, heads]. Packed context uses + request_ids; verification repeats generation tables rows_per_request times. + Both supply per-query kv_lens. Single-token decode uses live metadata lengths. + Incomplete/invisible pools receive the FP32 minimum before top-k. """ state = self._cache_state(metadata) - tables, gen_lens = self._rows( - state, request_index, q.shape[0], rows_per_request, request_ids=request_ids - ) + tables, gen_lens = self._rows(state, rows_per_request, request_ids=request_ids) capacity = state.block_tables.shape[1] * state.tokens_per_block kpool = self.sparse_params.index_kpool return kpool_score( @@ -622,7 +405,7 @@ def score_pools( precision="tf32", # Context rows: the query tokens of a request share its block # table, so a program gathers each pool-key block once for 16 rows. - rows_per_program=1 if request_index is None and request_ids is None else 16, + rows_per_program=1 if request_ids is None else 16, request_ids=request_ids, ) @@ -631,7 +414,6 @@ def expand_selection( selected: torch.Tensor, metadata, *, - request_index: int | None = None, kv_lens: torch.Tensor | None = None, rows_per_request: int = 1, request_ids: torch.Tensor | None = None, @@ -644,9 +426,7 @@ def expand_selection( row addressing as :meth:`score_pools`. """ state = self._cache_state(metadata) - tables, gen_lens = self._rows( - state, request_index, selected.shape[0], rows_per_request, request_ids=request_ids - ) + tables, gen_lens = self._rows(state, rows_per_request, request_ids=request_ids) _, base_row, rows_per_slot = latent_pool_rows(state.latent_pool) return kpool_expand( selected, @@ -672,15 +452,10 @@ def create_output( is_gen_only: bool = False, **kwargs, ) -> list[torch.Tensor]: - """Allocate the standard flat output buffer for this absorbed branch. - - Reconciles the inherited ``TrtllmAttention.create_output``, whose MLA - context leg allocates ``num_heads * v_head_dim``: the absorbed - formulation emits the *latent* width ``num_heads * kv_lora_rank`` in - **both** phases (the model layer applies the absorbed V projection to - it afterwards, so ``v_head_dim`` never appears at this boundary). - Quantized/NVFP4 output modes are not implemented on this branch and - are rejected loudly rather than silently mis-allocated. + """Allocate [tokens, num_heads * kv_lora_rank] for absorbed attention. + + Both phases emit latent-width outputs; the model applies the V projection. + Quantized outputs are unsupported. """ del metadata, attention_mask, is_gen_only, kwargs if is_quantize_output: @@ -751,14 +526,10 @@ def _dispatch_sparse_core( def _finalize_output( self, out_latent: torch.Tensor, output: torch.Tensor | None ) -> torch.Tensor: - """Flatten the kernel output to ``[T, num_heads * kv_lora]``. - - With no caller buffer the kernel's own (contiguous) allocation is - returned viewed flat -- same shape/dtype/device as - :meth:`create_output` would allocate, without a redundant copy. A - caller-provided ``forward_args.output`` (already validated in - :meth:`forward`) is written in place and returned, matching the - TRTLLM family's caller-owned-buffer semantics. + """Flatten to [tokens, num_heads * kv_lora_rank], filling output when supplied. + + Without a caller-owned buffer, flattening the contiguous head/feature axes + preserves the view even when token strides include padded query heads. """ if output is None: # The real heads remain contiguous within each token even when @@ -777,60 +548,33 @@ def forward( forward_args: AttentionForwardArgs | None = None, **kwargs, ) -> torch.Tensor: - """Standard ``AttentionBackend.forward`` for the k-pool sparse core. - - Arguments follow the base contract exactly: - - ``q`` - Absorbed latent-space queries, ``[T, num_heads * kv_lora]`` or the - equivalent ``[T, num_heads, kv_lora]`` view. - ``k`` - Context phase: the request's contiguous cached latent prefix - ``[KV, kv_lora]`` (``num_kv_heads == 1``), gathered from the same - metadata through :meth:`gather_paged_prefix`. Generation phase: - ``None`` -- the kernel reads the paged latent pool directly - through the storage row view derived from ``metadata``. - ``v`` - Must be ``None``: the latent rows serve as both K and V, exactly - as in absorbed MLA; the model layer applies the absorbed V - projection to the returned latent output. - ``metadata`` - The engine's prepared typed metadata. Required: the paged pools, - block tables, and visible lengths are derived from it (see - :meth:`_cache_state`); ``None`` or unprepared metadata is a loud - error. - ``forward_args`` / ``**kwargs`` - Merged through :func:`merge_attention_forward_args`, which - rejects unknown kwargs and forward_args/kwargs mixing. The - model layer's pool-expanded selection travels in the typed - ``sparse_backend_args.topk_indices`` field (int32 - ``[T, output_width]``, request-local positions, - :data:`INDEX_SENTINEL` padding), and - ``attention_input_type`` declares the phase -- this backend is - phase-explicit, so ``mixed`` is rejected. A caller-provided - ``forward_args.output`` is validated (shape/dtype/device), - written in place, and returned -- the TRTLLM family's - caller-owned-buffer semantics. Quantized-output modes - (``out_scale``/``out_scale_sf``/``output_sf``) are not - implemented here and are rejected loudly. - - Returns the flat latent-space attention output - ``[T, num_heads * kv_lora]`` -- the base contract's - ``(num_q_tokens, num_heads * head_dim)`` with this backend's - ``head_dim == kv_lora_rank``. The model layer views it per-head for - the absorbed V projection. + """Run sparse attention against paged latent rows. + + Args: + q: Absorbed queries [tokens, num_heads, kv_lora_rank], or the flattened + [tokens, num_heads * kv_lora_rank] view. + k: Must be None; KV rows come from the prepared cache metadata. + v: Must be None; latent rows serve as both K and V. + metadata: Prepared TRTLLM metadata with Glm5NextMamba2Metadata tables. + forward_args: Must provide topk_rows (int32 [tokens, width], -1 invalid) + and a context_only or generation_only phase. An optional flat output + buffer is validated and filled in place. Quantized outputs are unsupported. + **kwargs: Legacy argument carrier, merged by the shared base contract. + + Returns: + Latent output [tokens, num_heads * kv_lora_rank]. """ forward_args = merge_attention_forward_args(forward_args, kwargs) sparse_args = forward_args.sparse_backend_args - topk_indices = sparse_args.topk_indices if sparse_args is not None else None topk_rows = getattr(sparse_args, "topk_rows", None) - if topk_indices is None and topk_rows is None: - raise NotImplementedError( - "GlmKpoolSparseAttention needs the model layer's pool-expanded " - "selection in forward_args.sparse_backend_args.topk_indices; the " - "k-pool scoring/selection is module math -- see " - "modeling_glm5_next.Glm5NextSparseAttention." + if topk_rows is None: + raise ValueError( + "glm_kpool requires pool-expanded selection in sparse_backend_args.topk_rows" ) + if sparse_args.topk_indices is not None: + raise ValueError("glm_kpool accepts topk_rows, not request-local topk_indices") + if k is not None: + raise ValueError("glm_kpool reads paged latent rows from metadata; k must be None") if v is not None: raise ValueError("glm_kpool consumes latent rows as both K and V; v must be None") if q.dim() == 2: @@ -861,63 +605,12 @@ def forward( f"{output.device}" ) - state = self._cache_state(metadata) input_type = forward_args.attention_input_type - if topk_rows is not None: - # Fast path (either phase): the selection was expanded and - # translated to latent-pool row ids by expand_selection, so the - # kernel reads the paged pool directly; k must not be passed. - if k is not None: - raise ValueError("glm_kpool: topk_rows addresses the paged pool; k must be None") - if input_type not in ( - AttentionInputType.context_only, - AttentionInputType.generation_only, - ): - raise ValueError(f"glm_kpool forward is phase-explicit, got {input_type!r}") - kv_rows, _, _ = latent_pool_rows(state.latent_pool) - return self._finalize_output(self._dispatch_sparse_core(q, kv_rows, topk_rows), output) - if input_type == AttentionInputType.context_only: - if k is None: - raise ValueError( - "glm_kpool context forward needs the request's contiguous " - "latent prefix as k (see gather_paged_prefix)" - ) - if topk_indices is None: - raise ValueError( - "glm_kpool context forward selects over the request's own " - "contiguous prefix; pass request-local topk_indices, not rows" - ) - kv_len = k.shape[0] - out_latent = self._dispatch_sparse_core( - q, k.view(kv_len, 1, self.kv_lora_rank), topk_indices - ) - return self._finalize_output(out_latent, output) - if input_type == AttentionInputType.generation_only: - if k is not None: - raise ValueError( - "glm_kpool generation forward reads the paged latent pool " - "from metadata; k must be None" - ) - gen_tables = state.block_tables[state.num_contexts :] - num_gens = gen_tables.shape[0] - if num_gens == 0 or q.shape[0] % num_gens: - raise ValueError( - f"glm_kpool generation forward got {q.shape[0]} query rows for " - f"{num_gens} generation requests in the metadata" - ) - kv_rows, base_row, rows_per_slot = latent_pool_rows(state.latent_pool) - # Speculative verification packs ``1 + runtime_draft_len`` query - # rows per generation request, request-major; every row of a - # request resolves its selection through that request's table. - tokens_per_request = q.shape[0] // num_gens - if tokens_per_request > 1: - gen_tables = gen_tables.repeat_interleave(tokens_per_request, dim=0) - topk_rows = positions_to_pool_rows( - topk_indices, gen_tables, state.tokens_per_block, base_row, rows_per_slot - ) - return self._finalize_output(self._dispatch_sparse_core(q, kv_rows, topk_rows), output) - raise ValueError( - "glm_kpool forward is phase-explicit: set " - "forward_args.attention_input_type to context_only or " - f"generation_only, got {input_type!r}" - ) + if input_type not in ( + AttentionInputType.context_only, + AttentionInputType.generation_only, + ): + raise ValueError(f"glm_kpool forward is phase-explicit, got {input_type!r}") + state = self._cache_state(metadata) + kv_rows, _, _ = latent_pool_rows(state.latent_pool) + return self._finalize_output(self._dispatch_sparse_core(q, kv_rows, topk_rows), output) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py index 8a025ddcf7fd..189f006d8634 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py @@ -12,31 +12,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Cache manager and prepared metadata of the glm_kpool sparse backend. +"""GLM hybrid cache manager and persistent sparse-attention metadata. -``Glm5NextMamba2Metadata`` is the model's mamba metadata: the shared Kimi KDA -metadata (chunk schedule, aligned generation slot indices for the replay -verify kernel) plus the sparse layers' persistent buffers below. - -CUDA graphs replay captured kernels over fixed buffer addresses; nothing -Python re-runs at replay. Every per-step request-derived value the sparse -layers consume in decode must therefore live in persistent device buffers -refreshed by ``prepare()`` -- which the engine (and the CUDA-graph runner, -before every replay) calls outside the captured region. This subclass adds -exactly those buffers to the mamba metadata the hybrid manager already -attaches: - -* ``glm_block_tables`` -- ``[max_batch, max_blocks_per_seq]`` base-slot - page ids (pinned staging + one async H2D per step); -* ``glm_kv_lens`` -- ``[max_batch]`` per-request visible lengths - (cached + this step's tokens), same staging pattern; -* ``glm_cached_lens_host`` / ``glm_ctx_cu_seqlens`` -- plain host values - for the prefill path, which is never captured (decode-only graphs). - -The buffer width is fixed at first use from the manager's own -``max_blocks_per_seq`` so captured gathers can never need a wider table; -a mid-run widening request is a hard error rather than a silent -reallocation that stale graphs would keep reading. +Kimi KDA metadata handles recurrent state and replay scheduling. GLM adds raw +slot tables with fixed device addresses plus host prefill schedules. Visible +lengths come from TRTLLM kv_lens_cuda, including overlap and MTP corrections. """ from __future__ import annotations @@ -52,24 +32,13 @@ class Glm5NextMamba2Metadata(KimiK3MambaMetadata): - def __init__( - self, max_batch_size: int, chunk_size: int, max_num_tokens: int | None = None - ) -> None: - # The KDA chunk-index buffer holds one row per 64-token chunk; a - # harness that gives no token budget gets a generous fixed capacity - # (two int64 per row -- negligible). - super().__init__( - max_batch_size, chunk_size, max_num_tokens if max_num_tokens is not None else 65536 - ) + def __init__(self, max_batch_size: int, chunk_size: int, max_num_tokens: int) -> None: + super().__init__(max_batch_size, chunk_size, max_num_tokens) from tensorrt_llm._utils import prefer_pinned self._glm_pin = prefer_pinned() self.glm_block_tables: torch.Tensor | None = None self._glm_block_tables_cpu: torch.Tensor | None = None - self.glm_kv_lens = torch.zeros(max_batch_size, dtype=torch.long, device="cuda") - self._glm_kv_lens_cpu = torch.zeros( - max_batch_size, dtype=torch.long, pin_memory=self._glm_pin - ) self.glm_cached_lens_host: list[int] = [] self.glm_ctx_cu_seqlens: list[int] = [0] @@ -118,11 +87,6 @@ def prepare(self, attn_metadata) -> None: cu.append(cu[-1] + length) self.glm_ctx_cu_seqlens = cu - self._glm_kv_lens_cpu[:batch].copy_( - torch.as_tensor([c + n for c, n in zip(cached, lens)], dtype=torch.long) - ) - self.glm_kv_lens[:batch].copy_(self._glm_kv_lens_cpu[:batch], non_blocking=True) - width = int(getattr(manager, "max_blocks_per_seq", 0)) or 1 self._glm_ensure_tables(width) pages = manager.get_batch_slot_tables(list(request_ids)[:batch]) @@ -135,25 +99,12 @@ def prepare(self, attn_metadata) -> None: class Glm5NextCacheManager(MambaHybridCacheManagerV2): - """One KVCacheManagerV2 lifecycle for all three kinds of GLM state. - - The three state families have genuinely different shapes and are kept - that way -- none is padded into a faux common KV tensor: - - * linear-attention layers: the recurrent accumulator and the four-tap - convolution history, carried by the inherited Mamba side with - ``conv_state_layout='q_k_v'`` (the convolution's ``[q | k | v]`` - section order); - * sparse-attention layers: the ``kv_lora_rank``-wide compressed latent, - carried by the standard attention pages with ``SELFKONLY``; - * sparse-attention layers again: the indexer's packed ``[k | gate]`` - state, added here as one extra ``Role.INDEX_KEY`` buffer per sparse - layer. + """Manage KDA state, latent KV and indexer buffers in one V2 lifecycle. - The extra buffer is registered through the base class's own - ``_extra_buffers_per_layer`` hook, so allocation, block reuse, slot - release, and disaggregated bookkeeping keep working. A second manager - would duplicate request ownership and break exactly those lifecycles. + KDA uses the inherited recurrent/conv pools; sparse MLA uses SELFKONLY pages. + An extra BF16 INDEX_KEY buffer stores [k | gate | pool key] per sparse layer. + Registering it through _extra_buffers_per_layer shares allocation, reuse, + release and disaggregated transfer with the base manager. """ def __init__( @@ -191,14 +142,9 @@ def get_index_state_buffer(self, layer_idx: int) -> torch.Tensor | None: ) def _sparse_pool_id(self) -> int: - """The single V2 layer-group id that owns every sparse layer. + """Require one V2 layer group for the shared sparse-layer block table. - Base page indices are per layer group, so one block table can - address both the latent and the indexer views only because all - sparse layers -- whose KEY and INDEX_KEY buffers share each - layer's group -- resolve to one group. Asserted, not assumed: - a future geometry change that splits the group must fail here - rather than silently interleave two slot spaces. + KEY and INDEX_KEY views must use the same raw slot space across sparse layers. """ pools = { self.layer_to_pool_mapping_dict[self.layer_offsets[layer_id]] @@ -213,22 +159,10 @@ def _sparse_pool_id(self) -> int: return pools.pop() def get_batch_slot_tables(self, request_ids: Sequence[int]) -> list[list[int]]: - """Raw base-slot IDs per request, for the slot-major state views. + """Return raw base-slot IDs, without V2's per-layer page-index scaling. - ``get_batch_cache_indices`` is scaled for V2's flattened - per-layer page views -- it returns ``base * scale // kv_factor`` - (scale is the coalesced buffers-per-slot count; 11 on the real - checkpoint) -- so feeding its output to the slot-major views from - :meth:`get_latent_state_buffer` / :meth:`get_index_state_buffer` - addresses the wrong slots and eventually runs past the pool. - Those views are indexed by the *base* page id itself, so this - accessor requests the identity conversion from the same V2 - bookkeeping (``is_kv_aggregate=False, index_scale=1``). - - A pipeline-parallel rank whose local slice holds no sparse layer - (e.g. the first three GLM layers are all linear attention) has no - sparse pool at all; it gets empty rows, which no layer on that - rank ever reads. + Both latent and indexer views fold that scaling into their slot stride. + PP ranks without local sparse layers return empty rows. """ if not any(layer_id in self.layer_offsets for layer_id in self.sparse_layer_ids): return [[] for _ in request_ids] @@ -240,29 +174,11 @@ def get_batch_slot_tables(self, request_ids: Sequence[int]) -> list[list[int]]: ) def get_latent_state_buffer(self, layer_idx: int) -> torch.Tensor | None: - """Paged latent state for ``layer_idx``, addressed by **slot** id. - - ``get_buffers`` hands back a view whose dim-0 stride is a *single* - page, but V2 coalesces every layer's ``Role.KEY`` buffer into one - pool and starts layer ``L``'s view ``L`` pages into it. Indexing two - layers' views with the same block id therefore makes them overlap - almost entirely -- measured on this checkpoint, eleven sparse layers - share a 2046-page pool and each layer's page ``p`` is the next - layer's page ``p - 1``. - - V2's own callers never index that view with a raw block id: - ``_get_batch_cache_indices_by_pool_id`` multiplies every base page - index by ``get_layer_page_index_scale(layer_idx)`` first. Folding - that scale into the *view* rather than into each caller's block - table is exactly what :meth:`get_index_k_buffer` already does for - ``Role.INDEX_KEY`` (``[slots, scale, ...][:, 0]``), and it is the - only convention under which one block table can address both pools: - the coalesced INDEX_KEY view is expressible *only* slot-indexed. + """Return a slot-major [slots, tokens_per_block, num_kv_heads, head_dim] view. - Returns ``[num_slots, tokens_per_block, num_kv_heads, head_dim]``, - the same rank as :meth:`get_index_state_buffer`, so a caller - decomposes a position into one ``(slot, within_slot)`` pair and uses - it against both. + get_buffers exposes per-page strides. Coalesced layers instead share raw slot + IDs, so fold the page converter's scale into the slot stride, matching the + INDEX_KEY view. No payload is copied. """ pages = self.get_buffers(layer_idx) if pages is None: diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/params.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/params.py index febdf64a4910..43988dd43025 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/params.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/params.py @@ -103,13 +103,10 @@ def output_width(self) -> int: @dataclass(kw_only=True, slots=True) class GlmKpoolBackendForwardArgs(SparseBackendForwardArgs): - """``SparseBackendForwardArgs`` plus the backend's own row-id selection. + """Global latent-cache row selection produced by expand_selection. - ``topk_rows`` carries a selection already translated to latent-cache row - ids (int32 ``[T, kernel_output_width]``, ``-1`` invalid) by - :meth:`GlmKpoolSparseAttention.expand_selection`; when present it is - consumed directly and ``topk_indices`` (request-local positions) is not - needed. + topk_rows is int32 [tokens, kernel_output_width], with -1 marking invalid + entries. Request-local topk_indices from the base carrier are unsupported. """ topk_rows: torch.Tensor | None = None diff --git a/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py b/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py index 916b2e7a461d..47d2804c14e8 100644 --- a/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py +++ b/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py @@ -12,12 +12,9 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Checkpoint-key mapping for GLM-5.3-Flash (``glm5_next``). +"""Map GLM-5.3-Flash checkpoint keys to text/MTP destinations and loading owners. -Everything here is a pure function of checkpoint key names and the HF config: -the per-key audit (loaded / transformed / ignored), the key remap onto the -runtime's parameter names and destination-owner routing. The tensors are placed by -``Glm5NextForCausalLM.load_weights``. +Mapping and audit use key names and config only; the model loader places tensors. """ from __future__ import annotations @@ -36,11 +33,7 @@ from tensorrt_llm._torch.pyexecutor.config_utils import unwrap_glm5_next_text_config from tensorrt_llm.quantization.mode import QuantAlgo -#: Checkpoint namespaces this bring-up deliberately does not load. These are -#: matched as exact dotted-component prefixes, never as substrings or globs: a -#: pattern like ``*visual*`` would also swallow a decoder weight that merely -#: contained the word, and the whole point of the audit is that nothing is -#: dropped by accident. +# Match dotted namespaces exactly so unrelated decoder weights are not ignored. _VISION_PREFIX = "model.visual." _LANGUAGE_PREFIX = "model.language_model." @@ -60,8 +53,7 @@ class Disposition: #: Placed on a destination parameter unchanged. LOADED = "loaded" - #: Placed after a shape/dtype/layout transformation (conv fusion, expert - #: stacking, block-FP8 dequantization, or a companion scale tensor). + #: Routed through expert packing or scale-layout conversion. TRANSFORMED = "transformed" #: Deliberately not loaded, under an exact allowlisted namespace. IGNORED = "ignored" @@ -126,19 +118,11 @@ def audit_glm5_next_checkpoint( *, num_mtp_layers: int = 0, ) -> Glm5NextWeightAudit: - """Resolve every checkpoint key to exactly one destination and disposition. - - This is the Goal-1.2 contract in executable form. It is deliberately - analytic -- it needs only the safetensors index and the config, not 328 GB - of materialized weights -- so it can gate every later loading change - cheaply. - - ``num_mtp_layers`` is how many of the checkpoint's appended next-n - prediction (MTP) layers the model actually instantiates -- ``0`` for the - plain text model, ``1`` under one-model MTP speculative decoding. Those - layers' keys are placed on ``model.layers.{num_hidden_layers + i}.*`` - (the alias the speculative base class appends the draft layers under); - any remaining MTP layer stays an allowlisted ignore. + """Classify checkpoint keys and resolve text/MTP destinations without loading tensors. + + num_mtp_layers controls how many appended checkpoint layers are instantiated. + Remaining MTP layers and the separately loaded vision namespace are ignored; + unrecognized keys are reported as unresolved. """ text = unwrap_glm5_next_text_config(config) num_layers = int(text.num_hidden_layers) @@ -192,19 +176,9 @@ def audit_glm5_next_checkpoint( def glm5_next_is_quantized(model_config: ModelConfig[PretrainedConfig]) -> bool: - """Whether construction uses the checkpoint's block-FP8 form. - - The runtime constructs models through ``AutoModelForCausalLM.from_config``, - which calls ``cls(model_config)`` with no further arguments -- so the - quantization decision must live on the ``ModelConfig`` itself, exactly - where ``ModelConfig.from_pretrained`` puts it when it reads the - checkpoint's ``quantization_config`` (``weight_block_size=[128,128]`` maps - to ``FP8_BLOCK_SCALES``). A constructor flag that defaulted to bf16 would - make the runtime path build a model the loader must reject. - - This checkpoint is published in exactly one quantized form; any other - non-None algorithm on the config is a configuration error, not a request - for a different build. + """Read block-FP8 construction from ModelConfig, rejecting other quantization modes. + + Unquantized modules are also supported for component construction. """ quant = getattr(model_config, "quant_config", None) if quant is None or quant.quant_algo is None: @@ -235,16 +209,10 @@ def _destination_owner(dest: str, num_layers: int) -> Any: @register_mapper("HF", "Glm5NextForConditionalGeneration") @register_mapper("HF", "Glm5NextForCausalLM") class Glm5NextHfWeightMapper(HfWeightMapper): - """Checkpoint-key mapper for GLM-5.3-Flash (``glm5_next``). - - The HF checkpoint is a multimodal ``Glm5NextForConditionalGeneration`` - tree; the text model loads its ``model.language_model.*`` subtree with an - audited 1:1 placement (see :func:`audit_glm5_next_checkpoint`). This class - is the mapping half of that loader: which keys are ignored (vision tower, - surplus MTP layers), where each remaining key lands (:meth:`destination`), - and which materialization unit owns it (:meth:`owner`). The model's - ``load_weights`` owns the placement itself, so the generic module-name - callbacks of :class:`HfWeightMapper` are not used for this architecture. + """Resolve checkpoint destinations and owners for the model's custom loader. + + The text decoder loads model.language_model.* and optional MTP layers; vision + weights are handled separately. Generic module-name callbacks are not used. """ def audit(self, keys: Iterable[str]) -> Glm5NextWeightAudit: diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next.py b/tensorrt_llm/_torch/models/modeling_glm5_next.py index 6986e62d1ff2..003c8f8de4b2 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next.py @@ -40,7 +40,6 @@ AttentionMetadata, ) from ..attention.backends.sparse.glm_kpool import ( - Glm5NextCacheManager, Glm5NextMamba2Metadata, GlmKpoolBackendForwardArgs, GlmKpoolSparseParams, @@ -69,7 +68,6 @@ from ..distributed import AllReduceStrategy from ..modules.mamba.mamba2_metadata import Mamba2Metadata from ..modules.mhc.hyper_connection import mHC - from ..pyexecutor.kv_cache.mamba_cache_manager import MambaHybridCacheManagerV2 from .checkpoints.base_weight_mapper import BaseWeightMapper @@ -83,17 +81,6 @@ SPARSE_MLP = "sparse" -def get_glm5_next_text_config(config: PretrainedConfig) -> PretrainedConfig: - """Return the text-decoder config, accepting either nesting level. - - The runtime resolves the model from the top-level ``Glm5NextConfig``, but - every decoder contract (schedules, ranks, MoE, HC) lives on - ``text_config``. Callers may already hold the inner config, so this is - idempotent. - """ - return unwrap_glm5_next_text_config(config) - - @dataclass(frozen=True) class Glm5NextSchedule: """The two literal per-layer dispatch lists, validated against each other.""" @@ -113,15 +100,11 @@ def mlp_indices(self, kind: str) -> tuple[int, ...]: def resolve_glm5_next_schedule(config: PretrainedConfig) -> Glm5NextSchedule: - """Read and cross-validate the literal dispatch lists. + """Validate the explicit attention and MLP schedules. - Raises on any disagreement between the three redundant encodings. A model - whose attention schedule is inferred from a cadence, or whose MLP schedule - is inferred from ``first_k_dense_replace``, would silently place the wrong - module (and therefore the wrong cache descriptor) at some layer; the config - states both lists explicitly, so there is no reason to guess. + Cross-check optional redundant fields; do not infer layer types from a cadence. """ - text = get_glm5_next_text_config(config) + text = unwrap_glm5_next_text_config(config) num_layers = int(text.num_hidden_layers) attention = tuple(text.layer_types) @@ -141,9 +124,7 @@ def resolve_glm5_next_schedule(config: PretrainedConfig) -> Glm5NextSchedule: schedule = Glm5NextSchedule(attention=attention, mlp=mlp) - # Third, redundant encoding of the attention schedule. It is not used for - # dispatch, but a disagreement means the checkpoint is not the variant this - # bring-up was validated against. + # Optional redundant schedule fields must agree with the explicit lists. linear_attn_config = getattr(text, "linear_attn_config", None) or {} kda_layers = linear_attn_config.get("kda_layers") full_attn_layers = linear_attn_config.get("full_attn_layers") @@ -156,8 +137,7 @@ def resolve_glm5_next_schedule(config: PretrainedConfig) -> Glm5NextSchedule: "glm5_next linear_attn_config.full_attn_layers disagrees with layer_types" ) - # first_k_dense_replace is asserted against the literal list, not used to - # build it. + # Validate the dense prefix without using it to infer the MLP schedule. first_k_dense = getattr(text, "first_k_dense_replace", None) if first_k_dense is not None: expected_dense = tuple(range(int(first_k_dense))) @@ -172,19 +152,10 @@ def resolve_glm5_next_schedule(config: PretrainedConfig) -> Glm5NextSchedule: def glm5_next_allreduce_strategy() -> AllReduceStrategy: - """The ``AllReduceStrategy`` for every TP collective this model owns. - - Default ``ONESHOT``: the fused one-shot Lamport kernel (15-20 us for the - decode-sized [tokens, hidden] messages here, vs 20-75 us for NCCL's LL - ring at c=8..64; ``MIN_LATENCY`` switches to the two-shot kernel from a - few hundred tokens, which measured 38 us vs 19 us one-shot for the - 256-token speculative-verify messages). Messages above - :attr:`Glm5NextAllReduce.SMALL_MAX_TOKENS` go to NCCL regardless. The - bring-up pinned ``NCCL`` because the ``AUTO`` *autotuner* raced at TP4 - decode; a fixed strategy never enters the autotuner, so the race does not - apply. ``TLLM_GLM5_ALLREDUCE`` selects another strategy by name - (``NCCL``, ``MIN_LATENCY``, ``TWOSHOT``, ``AUTO``, ``NCCL_SYMMETRIC``) for - A/B measurement and as the escape hatch. + """Select the small-message TP all-reduce strategy (default: ONESHOT). + + TLLM_GLM5_ALLREDUCE overrides the strategy for tuning. Large messages use NCCL + regardless of this setting; a fixed strategy avoids AUTO's runtime autotuning. """ from ..distributed import AllReduceStrategy @@ -229,20 +200,12 @@ def glm5_next_attention_mapping(mapping: Mapping | None) -> Mapping | None: class Glm5NextAllReduce(nn.Module): - """TP all-reduce whose strategy follows the message size at call time. - - The fused one-shot Lamport kernel (``ONESHOT``) is 15-20 us for - decode-sized ``[tokens, hidden]`` messages where NCCL's LL ring takes - 20-75 us, but from ~1K tokens up the fused kernels are slower than NCCL - (56 vs 46 us at 1K tokens; 408 vs 355 us two-shot vs NCCL at 8K). One - ``AllReduce`` per strategy, chosen by the token count -- a Python int, so a - captured decode graph has a fixed choice. Parameter-free. + """Use the configured TP collective for small messages and NCCL for large ones. + + Dispatch depends only on tensor shape, so CUDA graph capture fixes the choice. """ - #: Messages with more tokens than this go to NCCL. Measured on this node - #: (TP4, hidden 4096, bf16): the fused kernels win up to ~256 tokens - #: (15-19 us vs 20-27 us NCCL), tie around 512, and lose from 1024 tokens - #: (56 vs 46 us) upward. + # TP4/BF16 tuning uses fused collectives up to 512 tokens, then NCCL. SMALL_MAX_TOKENS = 512 def __init__(self, mapping: Mapping, dtype: torch.dtype = torch.bfloat16) -> None: @@ -268,16 +231,11 @@ def forward(self, x: torch.Tensor) -> torch.Tensor: def _normalize_glm5_next_top_config(config: PretrainedConfig) -> None: - """Give the composite multimodal config the fields the runtime reads. - - The checkpoint's top-level ``Glm5NextConfig`` carries no - ``num_hidden_layers`` or ``torch_dtype`` -- both live on ``text_config`` -- - but ``DecoderModel``/``DecoderModelForCausalLM`` and executor capacity - planning read them from the config they are handed. Copy them up once, - from the text config, rather than teaching every runtime consumer about - the wrapper. Values already present are left alone. + """Expose text-config fields required by runtime capacity planning and MTP. + + Keep existing top-level values when the config is already flattened. """ - text = get_glm5_next_text_config(config) + text = unwrap_glm5_next_text_config(config) if getattr(config, "num_hidden_layers", None) is None: config.num_hidden_layers = int(text.num_hidden_layers) if getattr(config, "torch_dtype", None) is None: @@ -339,24 +297,16 @@ def build( @dataclass class Glm5NextRuntimeContext: - """Per-forward schedule arguments derived once from AttentionMetadata. - - The sparse layers take *schedule* values (request boundaries, positions) - plus the prepared ``metadata`` itself -- their attention backend derives - every cache pool, block table, and visible length from that metadata. The - KDA layers consume the prepared metadata directly (the shared Kimi KDA - mixer reads its pools and slot ids from it). This object is built - once per model forward; requests are packed context-first, matching the - executor's batch layout. + """Per-forward schedule shared by all layers of a context-first packed batch. + + Sparse backends derive cache state from metadata; KDA consumes it directly. """ - manager: Any num_contexts: int num_ctx_tokens: int num_generations: int ctx_cu_seqlens: list[int] cached_lens: list[int] - state_indices: torch.Tensor #: Per-request visible lengths (cached + this step's tokens) as a device #: tensor. The decode path consumes ONLY device values so that captured #: CUDA graphs replay against prepare()-refreshed buffers rather than @@ -386,37 +336,22 @@ def context_rows(self, kpool: int, device: torch.device) -> Glm5NextContextRows: @property def gen_phase(self) -> str: - """Which attention entry point the generation rows take. - - ``"decode"`` is the single-token path; ``"verify"`` is the - multi-token speculative verification path (the fused Kimi KDA replay - kernel: the state is committed after the golden token and the drafts - are cached for replay once the sampler has decided the accepted count). - """ + """Select single-token decode or multi-token speculative verification.""" return "decode" if self.gen_tokens_per_request == 1 else "verify" - def mixed_kwargs(self, layer_idx: int) -> dict[str, Any]: - """Arguments of the sparse layers' ``forward_mixed`` for a - context+generation batch. - - The attention module splits the packed tokens at ``num_ctx_tokens`` - and runs its context rows through the prefill kernels and its - generation rows through the decode or verify kernels; everything outside - attention runs once over the whole batch (the vLLM layout). The KDA - layers do this split inside the shared mixer from the metadata. - """ + def mixed_kwargs(self) -> dict[str, Any]: + """Split sparse attention by phase while keeping the rest of the batch together.""" return { "num_ctx_tokens": self.num_ctx_tokens, - "prefill": self.sparse_kwargs(layer_idx, "prefill"), - "generation": self.sparse_kwargs(layer_idx, self.gen_phase), + "prefill": self.sparse_kwargs("prefill"), + "generation": self.sparse_kwargs(self.gen_phase), "generation_phase": self.gen_phase, } - def sparse_kwargs(self, layer_idx: int, phase: str) -> dict[str, Any]: + def sparse_kwargs(self, phase: str) -> dict[str, Any]: # Schedule only: the backend (keyed by its own layer_idx) derives the # slot-indexed latent/indexer views, block tables, and lengths from # the prepared metadata itself. - del layer_idx kwargs: dict[str, Any] = {"metadata": self.metadata} if phase == "prefill": kwargs.update( @@ -441,11 +376,7 @@ def _glm5_gen_tokens_per_request(attn_metadata: AttentionMetadata, num_generatio """ if num_generations <= 0: return 1 - num_tokens = getattr(attn_metadata, "num_tokens", None) - if num_tokens is None: - # Harness carriers without the runtime's cached token count: the - # host seq_lens carry the same information (never a captured path). - num_tokens = int(attn_metadata.seq_lens.sum()) + num_tokens = attn_metadata.num_tokens gen_tokens = int(num_tokens) - int(attn_metadata.num_ctx_tokens) if gen_tokens <= 0 or gen_tokens % num_generations: raise ValueError( @@ -455,155 +386,56 @@ def _glm5_gen_tokens_per_request(attn_metadata: AttentionMetadata, num_generatio return gen_tokens // num_generations -def glm5_next_visible_lens(attn_metadata: AttentionMetadata, batch: int) -> torch.Tensor | None: - """Per-request visible lengths (``cached + this step's tokens``) as int64. - - Prefers the attention metadata's own ``kv_lens_cuda`` over the - ``Glm5NextMamba2Metadata`` copy. Both hold the same values after - ``prepare()``, but only ``kv_lens_cuda`` receives the engine's in-graph - corrections: under the overlap scheduler with speculative decoding the - host prepares generation requests as if every draft of the previous step - had been accepted, and ``_preprocess_inputs`` subtracts the rejected - count on device (``previous_kv_lens_offsets_cuda``) right before the - forward; the speculative worker likewise rewinds it between draft steps. - Reading the host-derived copy there positions the new latent/indexer rows - past the real prefix and attends stale page contents -- observed as - non-deterministic MTP output under config E. Returns ``None`` when the - metadata carries no ``kv_lens_cuda`` (harness carriers), so callers fall - back to the GLM buffer. The int32 -> int64 cast is a device op with no - host sync, so it is legal inside CUDA-graph capture. - """ - live = getattr(attn_metadata, "kv_lens_cuda", None) - if live is None: - return None - return live[:batch].to(torch.long) - - -def build_glm5_next_runtime_context( - attn_metadata: AttentionMetadata, - *, - kv_lens_source: str = "glm", -) -> Glm5NextRuntimeContext: - """Derive the per-forward cache arguments from prepared metadata. - - Requires ``attn_metadata.prepare()`` to have run: that is what attaches - ``mamba_metadata`` (the manager is a ``BaseMambaCacheManager``) and fills - its batch-ordered ``state_indices``. ``cached_lens`` follows the runtime's - own convention -- tokens already in the cache, excluding the ones in this - step -- which is exactly what ``forward_prefill``/``forward_decode`` seed - and position from. - - Visible lengths come from :func:`glm5_next_visible_lens` (the metadata's - device-corrected ``kv_lens_cuda``) whenever the metadata carries it, for - both the target and the MTP draft layer; the ``Glm5NextMamba2Metadata`` - copy is the fallback for harness carriers. ``kv_lens_source`` is kept for - call-site documentation (``"metadata"`` marks the draft layer, whose - lengths the speculative worker rewinds in place between draft steps) and - to reject unknown values; it no longer changes the source when - ``kv_lens_cuda`` is present. +def build_glm5_next_runtime_context(attn_metadata: AttentionMetadata) -> Glm5NextRuntimeContext: + """Derive per-forward schedules from prepared GLM metadata. + + Prefill uses host schedules prepared before forward. Decode and MTP use + kv_lens_cuda, which receives overlap corrections and draft rewinds on device. + No cache tables or lengths are reconstructed inside forward. """ - manager = attn_metadata.kv_cache_manager - if manager is None: + if attn_metadata.kv_cache_manager is None: raise ValueError("glm5_next requires a kv cache manager; got None") mamba_metadata = attn_metadata.mamba_metadata if mamba_metadata is None or mamba_metadata is False: - raise ValueError( - "glm5_next requires mamba_metadata; call attn_metadata.prepare() " - "with the Glm5NextCacheManager attached" + raise ValueError("glm5_next requires mamba_metadata; call attn_metadata.prepare() first") + if getattr(mamba_metadata, "glm_block_tables", None) is None: + raise RuntimeError( + "glm5_next requires prepared glm_block_tables; call attn_metadata.prepare() " + "with Glm5NextMamba2Metadata before eager execution or CUDA graph capture" ) - if kv_lens_source not in ("glm", "metadata"): - raise ValueError(f"glm5_next: unknown kv_lens_source {kv_lens_source!r}") + live_lengths = getattr(attn_metadata, "kv_lens_cuda", None) + if live_lengths is None: + raise ValueError("glm5_next requires prepared attn_metadata.kv_lens_cuda") batch = int(attn_metadata.seq_lens.shape[0]) num_contexts = int(attn_metadata.num_contexts) num_generations = batch - num_contexts - gen_tokens_per_request = _glm5_gen_tokens_per_request(attn_metadata, num_generations) - - if getattr(mamba_metadata, "glm_block_tables", None) is not None: - # Persistent path: every tensor below is a prepare()-refreshed buffer - # slice, so this function does no allocation, no H2D, and no host - # sync -- it is safe to run inside CUDA graph capture, and replays - # read the refreshed values at the same addresses. - kv_lens = glm5_next_visible_lens(attn_metadata, batch) - if kv_lens is None: - if kv_lens_source == "metadata": - raise ValueError( - "glm5_next draft layer needs attn_metadata.kv_lens_cuda (the " - "TRTLLM metadata family); the attached metadata has none" - ) - kv_lens = mamba_metadata.glm_kv_lens[:batch] - return Glm5NextRuntimeContext( - manager=manager, - num_contexts=num_contexts, - num_ctx_tokens=int(attn_metadata.num_ctx_tokens), - num_generations=num_generations, - ctx_cu_seqlens=mamba_metadata.glm_ctx_cu_seqlens, - cached_lens=mamba_metadata.glm_cached_lens_host, - state_indices=mamba_metadata.state_indices[:batch], - kv_lens=kv_lens, - metadata=attn_metadata, - gen_tokens_per_request=gen_tokens_per_request, - ) - - # Legacy eager construction, kept for harnesses whose fake managers do - # not attach the GLM metadata buffers. It allocates and copies, so it - # must never run inside a captured region. (The sparse backend applies - # the same rule to its own metadata-derived block tables.) - if getattr(attn_metadata, "is_cuda_graph", False): - raise RuntimeError( - "glm5_next CUDA-graph execution requires the Glm5NextCacheManager's " - "Glm5NextMamba2Metadata (persistent prepare()-refreshed buffers); " - "the attached mamba_metadata has no glm_block_tables" - ) - lens = attn_metadata.seq_lens.tolist() - kv_params = attn_metadata.kv_cache_params - if kv_params is None or kv_params.num_cached_tokens_per_seq is None: - raise ValueError("glm5_next requires kv_cache_params.num_cached_tokens_per_seq") - cached_lens = [int(n) for n in kv_params.num_cached_tokens_per_seq[:batch]] - - ctx_cu = [0] - for length in lens[:num_contexts]: - ctx_cu.append(ctx_cu[-1] + int(length)) - - device = torch.device("cuda", torch.cuda.current_device()) - kv_lens = torch.as_tensor( - [c + n for c, n in zip(cached_lens, lens)], dtype=torch.long, device=device - ) - - live = glm5_next_visible_lens(attn_metadata, batch) - if live is not None: - kv_lens = live - return Glm5NextRuntimeContext( - manager=manager, num_contexts=num_contexts, num_ctx_tokens=int(attn_metadata.num_ctx_tokens), num_generations=num_generations, - ctx_cu_seqlens=ctx_cu, - cached_lens=cached_lens, - state_indices=mamba_metadata.state_indices[:batch], - kv_lens=kv_lens, + ctx_cu_seqlens=mamba_metadata.glm_ctx_cu_seqlens, + cached_lens=mamba_metadata.glm_cached_lens_host, + kv_lens=live_lengths[:batch].to(torch.long), metadata=attn_metadata, - gen_tokens_per_request=gen_tokens_per_request, + gen_tokens_per_request=_glm5_gen_tokens_per_request(attn_metadata, num_generations), ) @register_auto_model("Glm5NextForCausalLM") class Glm5NextForCausalLM(SpecDecOneEngineForCausalLM): - """GLM-5.3-Flash text decoder with an optional one-model MTP drafter. + """GLM-5.3-Flash decoder with an optional one-model MTP drafter. - The speculative base class owns logits processing and the draft/verify - lifecycle. This class narrows the composite config and loads each rank's - checkpoint shard, including the optional appended MTP layer. Vision weights - are excluded explicitly by :func:`audit_glm5_next_checkpoint`. + The speculative base class manages draft/verify execution. The weight mapper + routes text and MTP tensors and excludes the vision namespace. """ @property def mamba_metadata_cls(self) -> type[Mamba2Metadata]: """Metadata with paged tables refreshed before CUDA graph replay.""" - return glm5_next_mamba_metadata_cls() + return Glm5NextMamba2Metadata def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: - text_config = get_glm5_next_text_config(model_config.pretrained_config) + text_config = unwrap_glm5_next_text_config(model_config.pretrained_config) # tie_word_embeddings is false on this checkpoint, so lm_head is a real # weight rather than a view of the embedding; it is asserted, not assumed. if bool(getattr(text_config, "tie_word_embeddings", False)): @@ -642,9 +474,7 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: # AutoModelForCausalLM.from_config calls cls(model_config) and nothing # else, so this is the only place the decision can live. self.quantized = glm5_next_is_quantized(model_config) - # One provenance line per rank: engine-scale runs (LLM API / serving) - # spawn MPI workers whose model objects the driver cannot introspect, - # so the resolved production stack is published through the worker log. + # Log the resolved backends from each worker process. attn_backends = sorted( { type(layer.self_attn.attn_backend).__name__ @@ -705,23 +535,12 @@ def apply_quant_config_exclude_modules(self) -> None: ) def infer_max_seq_len(self) -> int: - """Max sequence length the runtime sizes KV/mamba caches for. - - The executor calls this during capacity planning. GLM-5.3-Flash declares - ``max_position_embeddings=1048576`` and is fully NoPE (no rope-factor - scaling), so the value is the text config's directly. - """ + """Use the text config's context limit directly; this model has no RoPE scaling.""" return int(self.text_config.max_position_embeddings) @classmethod def get_preferred_kv_cache_manager_version(cls, pretrained_config=None) -> str: - """Opt this model into ``KVCacheManagerV2``. - - The hybrid latent-KV + pool-indexer + recurrent/conv state is owned by a - single ``Glm5NextCacheManager`` (a ``MambaHybridCacheManagerV2`` subclass, - :func:`glm5_next_cache_manager_cls`); V1 cannot express it. This is the - ``"auto"`` -> V2 resolution hook the runtime consults. - """ + """Use V2 for the shared latent-KV, pool-indexer and recurrent-state cache.""" return "V2" def attention_type(self, layer_idx: int) -> str: @@ -747,33 +566,18 @@ def load_weights( *, device_map: dict[Any, Any] | None = None, ) -> None: - """Materialize and fill the whole text model from a raw checkpoint. - - ``weight_mapper`` is the :class:`Glm5NextHfWeightMapper` registered for - this architecture (the runtime's ``ModelLoader`` hands over the - initialized instance; harnesses may omit it and one is created here). - It owns every checkpoint-key decision -- the audit, the key remap and - the destination owner -- while this method owns the materialization: - exact-shape placement one owner at a time, then the decode fusions. - A generic HF mapper is rejected: its module-name rules cannot place - this checkpoint. - - ``weights`` is any mapping from checkpoint key to the tensor **as - stored** -- e4m3 payloads and their FP32 block scales are copied - verbatim, never dequantized, because excluded modules are published in - BF16 and quantized ones in e4m3 with a scale, with no overlap. That - makes the load a 1:1 placement and keeps the resident model at the - checkpoint's own 328 GB. - - ``device_map`` maps each owner -- a layer index, or ``"embed"``, - ``"norm"``, ``"head"`` -- to a device. The model is expected to have - been constructed on ``meta``: each owner is materialized directly onto - its target device and filled immediately, so peak memory is one layer - above the final footprint rather than a second full copy. - - A checkpoint tensor that finds no parameter, and a parameter that - receives no tensor, are both errors: either one leaves a model that - still runs and still looks plausible. + """Load checkpoint tensors one owner at a time to bound peak memory. + + The registered GLM mapper resolves keys and destination owners. FP8 payloads + and FP32 block scales retain their checkpoint representation; module loaders + handle sharding and fused projections. Unresolved or unfilled tensors raise. + + Args: + weights: Mapping of checkpoint keys to tensors or lazy safetensors slices. + weight_mapper: Initialized Glm5NextHfWeightMapper, or None to create one. + device_map: Optional owner-to-device map. Owners are layer indices or + "embed", "norm", and "head". Modules are materialized from meta + directly onto their target devices. """ if weight_mapper is None: weight_mapper = Glm5NextHfWeightMapper() @@ -788,7 +592,7 @@ def load_weights( "glm5_next whole-model loading requires the block-FP8 build " "(quantized=True). Dequantizing all 288 experts of 42 routed " "layers to bf16 would double the resident model to ~656 GB and " - "move it four times further from the source's own arithmetic." + "exceed the supported loading footprint." ) # Decoder stack plus the aliased MTP draft layer(s), if any: owners # ``45..`` are the draft layers, placed from the checkpoint's own @@ -1283,55 +1087,16 @@ def packed_state(self, hidden_states: torch.Tensor) -> torch.Tensor: class Glm5NextSparseAttention(nn.Module): - """Fully NoPE sparse MLA with a pool-compressed indexer. - - ``qk_rope_head_dim`` is 0 on this checkpoint and ``mla_use_nope`` is true: - there is no text rotary call and no rotary cache. ``indexer_rope_interleave`` - is present in the config but vestigial for the text path, so no rotary - branch is created for it -- long-range position sensitivity comes from the - causal KDA layers and the indexer's learned pool APE. - - Layering follows the attention developer guide. This module owns the - module math only: low-rank q/kv projections and norms, the pool indexer's - scoring/selection (model-layer sparse prediction, as in MiniMax-M3), the - absorbed-MLA query/value reassociation, and ``o_proj``. Everything below - that -- the paged latent/indexer cache path and the sparse-MLA core -- - belongs to ``self.attn_backend``, a - :class:`~tensorrt_llm._torch.attention.backends.sparse.glm_kpool.GlmKpoolSparseAttention`: - a ``TrtllmAttention`` subclass (the fully-NoPE branch of the TRTLLM sparse - family) constructed through the standard ``create_attention(...)`` - dispatch on the configured backend slot (``ModelConfig.attn_backend``, - default TRTLLM) with ``SparseParams(algorithm="glm_kpool")``. Its typed - metadata family is ``TrtllmAttentionMetadata`` (``attn_backend.Metadata``), - the class the engine constructs for this model. - - Cache ownership: one latent ``kv_lora_rank``-wide entry per token (the - pre-``kv_b_proj`` latent, not expanded K/V) plus the indexer's packed - ``[k | gate]`` state, both held by the one hybrid ``KVCacheManagerV2`` and - read/written only by the backend, which derives every pool, block table, - and visible length from the prepared attention metadata it is handed -- - this module passes schedule values and the metadata, never raw pools. The - pool-expanded selection travels to the backend inside the standard - ``AttentionForwardArgs.sparse_backend_args`` carrier. - - ``kv_b_proj`` is on this checkpoint's ``modules_to_not_convert`` list - (BF16), so absorption is a reassociation of the same BF16 weights, not a - dequantization. - - Tensor parallelism: with ``tp_size > 1`` this module owns - ``num_heads = 64 // tp_size`` local query heads and the matching rows of - the column-sharded ``q_b_proj``/``kv_b_proj`` (so the absorbed per-head - views are local by construction), while the low-rank latents - (``q_a``/``kv_a``) and both norms stay replicated and the row-sharded - ``o_proj`` returns a partial that this module reduces once through - :class:`Glm5NextAllReduce` -- the DeepSeek-V3 MLA ownership with a - message-size-aware collective. The latent cache and the indexer's packed state stay - *complete* (512- and 256-wide) on every rank: ``num_kv_heads == 1`` is - never divided, all ranks compute identical latent/packed rows from the - replicated projections, and each rank's backend reads its own full copy. - The backend is constructed with the local head count, exactly as MLA's - per-rank ``num_heads // tp_size``. At ``tp_size == 1`` construction and - math are byte-identical to the pre-TP module. + """NoPE sparse MLA with a pool-compressed indexer. + + Owns projections, sparse selection and MLA weight absorption. The TRTLLM + sparse backend owns paged-cache access and attention, receiving metadata and + selected rows through AttentionForwardArgs. BF16 kv_b_proj weights are + reassociated directly, without dequantization. + + TP shards query heads and output projections; latent and indexer state stay + replicated. The output projection returns a partial for one explicit + reduction. Attention DP instead keeps all heads local with no TP reduction. """ #: The two low-rank input projections are both block-FP8 and replicated: @@ -1373,7 +1138,7 @@ def __init__( if self.qk_rope_head_dim != 0: raise ValueError( "glm5_next text attention is fully NoPE; a non-zero qk_rope_head_dim " - f"({self.qk_rope_head_dim}) would need a rotary path this bring-up does not have" + f"({self.qk_rope_head_dim}) is not supported" ) self.scaling = self.qk_head_dim**-0.5 eps = float(config.rms_norm_eps) @@ -1513,28 +1278,22 @@ def _select_and_attend_paged( visible: torch.Tensor, metadata: AttentionMetadata, input_type: AttentionInputType, - request_index: int | None = None, rows_per_request: int = 1, reduce: bool = True, request_ids: torch.Tensor | None = None, ) -> torch.Tensor: - """Fused indexer selection over the paged cache, then sparse attention. - - Score the cached pool keys, top-k, expand + tail + row translation, - attend. ``visible[i]`` is query row ``i``'s visible length (its own - position + 1); ``request_index`` selects one context request's block - table, ``request_ids`` (``[rows]`` int32) the block table of each - packed context row (``None`` for both: the generation rows). Every step is a fixed-shape - kernel with work proportional to the visible length, not the buffer - capacity, so decode replays it inside CUDA graphs and prefill no - longer materializes ``[tokens, pools, head_dim]`` gathers. + """Score pools, select members and tail, then attend to paged latent rows. + + visible gives each query's prefix length. Packed context uses request_ids; + verification uses rows_per_request. Single-token decode reads live lengths + from metadata. All device work uses fixed buffer shapes for graph replay. """ indexer = self.indexer rows = q_resid.shape[0] q_index = indexer.wq_b(q_resid).view(rows, indexer.n_heads, indexer.head_dim) # Generation rows read their visible length from the metadata unless # each request contributes several rows (context or verification). - plain_decode = request_index is None and request_ids is None and rows_per_request == 1 + plain_decode = request_ids is None and rows_per_request == 1 kv_lens = None if plain_decode else visible scores = self.attn_backend.score_pools( q_index, @@ -1542,7 +1301,6 @@ def _select_and_attend_paged( metadata, q_scale=indexer.softmax_scale, w_scale=indexer.head_mix_scale, - request_index=request_index, kv_lens=kv_lens, rows_per_request=rows_per_request, request_ids=request_ids, @@ -1559,7 +1317,6 @@ def _select_and_attend_paged( topk_rows = self.attn_backend.expand_selection( selected, metadata, - request_index=request_index, kv_lens=kv_lens, rows_per_request=rows_per_request, request_ids=request_ids, @@ -1585,23 +1342,11 @@ def forward_prefill( reduce: bool = True, ctx_rows_fn: Callable[[int, torch.device], Glm5NextContextRows] | None = None, ) -> torch.Tensor: - """Context phase, including continuation chunks -- all requests at once. - - ``cached_lens[i]`` is how many tokens of request ``i`` are already in - the cache, so a chunk is scored against the whole visible prefix rather - than only against its own tokens. Scoring a chunk in isolation is the - classic chunked-prefill bug here: it still passes a one-shot test. - Only schedule values are passed here; every pool write and read goes - through the backend's metadata-derived cache path. - - The packed context tokens of every request go through each stage in - one launch (projections, cache write, pool-key refresh, scoring, - top-k, expansion, attention): the kernels address each row's block - table through :class:`Glm5NextContextRows.request_ids`. A per-request - Python loop here was the dominant cost of mixed context+generation - iterations (measured 50% GPU idle with ~17 short prompts in flight). - ``ctx_rows_fn`` (the runtime context's cache) shares the schedule - across the sparse layers of one forward. + """Process all packed context rows, including continuation chunks. + + cached_lens counts tokens before this chunk. Each query attends to its full + visible prefix. Row schedules identify request block tables and completed + pools; ctx_rows_fn caches these schedules across layers of one forward. """ kpool = self.indexer.index_kpool device = hidden_states.device @@ -1642,34 +1387,14 @@ def forward_decode( metadata: AttentionMetadata, reduce: bool = True, ) -> torch.Tensor: - """Generation phase: one token per request, fully batched. - - CUDA-graph contract: every shape here is a function of the *buffer* - geometry (the metadata's block-table width times tokens_per_block), - never of the current lengths, and every request-dependent value - (``kv_lens`` and the metadata's block tables) is a device tensor - refreshed by metadata ``prepare()`` outside the captured region. No - ``.item()``/``.tolist()``, no per-request Python loop, no - host->device copy, and no data-dependent branch runs on this path, so - a captured decode graph replays correctly as lengths grow and slots - are reused. - - ``kv_lens[i]`` is the request's visible length *including* the token - being decoded, so the new token's position is ``kv_lens[i] - 1``; it - must be the same prepare()-refreshed lengths the metadata carries, - sliced to the generation rows. Positions at or beyond a request's - ``kv_lens`` gather page-0 garbage in the indexer prefix; the backend - masks them by replacement and the indexer's own validity masks exclude - them from pools, selection, and the tail. The attention core never - gathers the latent at all: the backend reads the paged pool directly - through its storage row view derived from the metadata, and only - selected (valid) positions are translated into row ids -- sentinels - stay ``-1``. There is no empty-row assertion here: it would force a - host sync, which is illegal under CUDA-graph capture -- and a decode - query is always covered by construction, either by the - always-selected tail (``visible % kpool != 0``) or by the final - complete pool, whose last member *is* the query position - (``visible % kpool == 0``). + """Decode one token per request using device-resident visible lengths. + + kv_lens includes the new token, whose position is kv_lens - 1. Shapes depend + only on buffer geometry; metadata preparation refreshes device tensors before + CUDA graph replay. This path must not synchronize lengths to the host. + + The backend masks invalid cache positions. Every query is covered by either + the incomplete tail pool or its final complete pool. """ batch = hidden_states.shape[0] positions = kv_lens - 1 # [B] @@ -1718,24 +1443,12 @@ def forward_verify( tokens_per_request: int, reduce: bool = True, ) -> torch.Tensor: - """Generation phase with ``tokens_per_request`` tokens per request. - - The speculative-decoding target scores the golden token plus the - drafts in one pass; the MTP draft layer's first step sees the same - packed layout. Request ``i``'s tokens sit at cache positions - ``kv_lens[i] - T .. kv_lens[i] - 1`` (``kv_lens`` already counts them, - exactly as in :meth:`forward_decode`), so the latent/indexer rows are - appended there and every query is scored against its *own* visible - prefix: pool scoring masks pools whose last member lies beyond the - query's position, and selection expansion builds the tail from the query's - own position, so token ``j`` never sees tokens ``j+1..``. Rows written - for drafts that are later rejected are simply overwritten by the next - step, which re-appends at the rewound ``kv_lens`` -- the same - positional-cache convention MLA relies on. - - Every shape is a function of ``tokens_per_request`` and the buffer - geometry, with no host sync, so a captured verification graph replays - against refreshed lengths and tables. + """Verify packed [batch * tokens_per_request, hidden] generation rows. + + Request tokens occupy positions kv_lens - T through kv_lens - 1. Each query + uses its own visible prefix, excluding later drafts. Rejected draft entries + are overwritten after lengths rewind. Shapes depend only on buffer geometry + and T, allowing CUDA graph replay with refreshed lengths and block tables. """ tokens_per_request = int(tokens_per_request) if tokens_per_request <= 1: @@ -1777,21 +1490,6 @@ def forward_verify( ) -# --------------------------------------------------------------------------- -# Heterogeneous request state -# --------------------------------------------------------------------------- - - -def glm5_next_mamba_metadata_cls() -> type[Mamba2Metadata]: - """Return the model's ``Mamba2Metadata`` subclass (see the sparse backend's ``cache_manager``).""" - return Glm5NextMamba2Metadata - - -def glm5_next_cache_manager_cls() -> type[MambaHybridCacheManagerV2]: - """Return the model's ``KVCacheManagerV2`` subclass (see the sparse backend's ``cache_manager``).""" - return Glm5NextCacheManager - - class Glm5NextGate(DeepseekV3Gate): """The shared DeepSeek noaux_tc gate, kept FP32 end to end. @@ -1836,20 +1534,11 @@ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: class Glm5NextMoE(nn.Module): - """Routed experts plus one always-active shared expert. - - The routed experts are a fused-MoE layer built through ``create_moe`` -- - the one selection entry point -- with the DeepSeek noaux_tc routing method - (:class:`Glm5NextGate`, the shared DeepSeek gate kept FP32) and the - DSV4-style uniform ``swiglu_limit_scalar``. On this checkpoint (FP8 block - scales, SM100) the resolver lands on ``TRTLLMGenFusedMoE`` whose - ``trtllm::fp8_block_scale_moe_runner`` consumes the clamp limit as - ``gemm1_clamp_limit``; the ``AUTO`` ``moe_backend`` default resolves to - ``TRTLLM`` for exactly this quant/SM pair in - ``ModelConfig.resolve_moe_backend``. The shared expert is the shared - ``GatedMLP`` (DeepSeek-V3 composition): TP-sharded with - ``reduce_output=False``, its partial summed with the routed partial before - this module's single all-reduce. + """Fused routed experts with a shared GatedMLP expert. + + Uses FP32 DeepSeek routing and clamped SwiGLU. Under TP, routed and shared + expert partials are summed before one reduction. Under attention DP, fused + MoE owns dispatch/combine and the shared expert is replicated. """ def __init__( @@ -2030,21 +1719,12 @@ def glm5_next_hyper_head(hidden_streams: torch.Tensor) -> torch.Tensor: class Glm5NextDecoderLayer(DecoderLayer): - """One decoder layer: two hyper-connection sites wrapping attention and FFN. - - The residual path is *not* an ordinary add. Each site collapses the four - streams into one sequence with the learned ``pre`` weights, runs the - sublayer, then writes the result back across the streams as - ``post * out + comb^T @ residual``. Both module choices come from the two - literal per-layer lists, never from a cadence or ``first_k_dense_replace``. - - Two entry points share one implementation: the runtime ``forward`` - receives ``AttentionMetadata`` (plus the once-per-forward - :class:`Glm5NextRuntimeContext`) and derives this layer's cache - arguments; ``forward_direct`` takes them explicitly and is what the - component tests call. The runtime path is - a thin argument-derivation shim over the direct path, so parity between - them is an argument-sourcing check, not a second implementation. + """Wrap attention and FFN in separate mHC residual connections. + + Each connection collapses [tokens, hc_mult, hidden] streams for the sublayer + and mixes its output back into the streams. Attention and MLP types come from + the explicit per-layer schedules. forward derives runtime arguments; + forward_direct applies the same math with explicit attention arguments. """ def __init__( @@ -2142,14 +1822,14 @@ def forward( hidden_states, phase="mixed", all_rank_num_tokens=all_rank_num_tokens, - **runtime_ctx.mixed_kwargs(self.layer_idx), + **runtime_ctx.mixed_kwargs(), ) if runtime_ctx.num_contexts > 0: return self.forward_direct( hidden_states, phase="prefill", all_rank_num_tokens=all_rank_num_tokens, - **derive(self.layer_idx, "prefill"), + **derive("prefill"), ) # "decode" (one token per request) or "verify" (golden + drafts per # request while a speculative-decoding target scores them). @@ -2158,7 +1838,7 @@ def forward( hidden_states, phase=phase, all_rank_num_tokens=all_rank_num_tokens, - **derive(self.layer_idx, phase), + **derive(phase), ) def forward_direct( @@ -2212,7 +1892,7 @@ class Glm5NextModel(DecoderModel): def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: _normalize_glm5_next_top_config(model_config.pretrained_config) super().__init__(model_config) - config = get_glm5_next_text_config(model_config.pretrained_config) + config = unwrap_glm5_next_text_config(model_config.pretrained_config) schedule = resolve_glm5_next_schedule(model_config.pretrained_config) # Pipeline parallelism rides the base machinery wholesale: the # inter-layer activation is the four-stream tensor [tokens, hc_mult, @@ -2328,7 +2008,7 @@ def _forward_single_phase_fused_hc( attn_out = layer.self_attn(x, runtime_ctx.metadata) else: attn_out = getattr(layer.self_attn, f"forward_{phase}")( - x, **runtime_ctx.sparse_kwargs(layer_idx, phase) + x, **runtime_ctx.sparse_kwargs(phase) ) residual, post, comb, x = layer.hc_ffn.fused_hc( attn_out, @@ -2418,31 +2098,16 @@ def forward( class Glm5NextMTP(nn.Module): - """GLM-5.3-Flash's one next-token-prediction layer (``layers.45``). - - Structure, verified against the checkpoint's own keys and the vLLM GLM-5 - port (``glm5next/nvidia/mtp.py``) since the HF reference implements no - MTP: ``enorm(embed(next_token)) ++ hnorm(target_hidden) -> eh_proj`` into - a **plain-residual** decoder block -- the MTP layer has no - hyper-connection weights, unlike the 45 main layers -- made of the same - sparse-MLA + k-pool indexer attention and 288+1-expert MoE as the main - sparse layers, closed by ``shared_head.norm``. The attention and MoE are - the *same classes* as the main stack (with ``layer_idx = 45``), so the - draft layer owns its own latent/indexer pages in the one hybrid cache - manager (the executor appends one attention layer for it) and shares the - projection-swap, TP-shard, and exact-placement loading contracts. - - Constructed by :class:`~.modeling_speculative.MTPForCausalLM` through - the model-type dispatch and called by the one-model speculative worker - as ``mtp_layer(input_ids, position_ids, hidden_states, embed_tokens, - attn_metadata, all_rank_num_tokens, spec_metadata)``. ``hidden_states`` - are the target's post-final-norm states for the same packed tokens (the - convention the DeepSeek-V3/Qwen3-Next/vLLM MTP paths all use), and the - first draft step's token schedule is *identical* to the target - verification pass (contexts: prompt shifted by one; generation: - ``1 + runtime_draft_len`` accepted/padded tokens at the same positions), - so it reuses the target's runtime-context derivation, reading the - metadata's live ``kv_lens_cuda`` so later draft steps' rewinds are seen. + """Native next-token prediction layer with ordinary residual connections. + + Normalizes and concatenates token embeddings and target hidden states before + eh_proj. Reuses the main stack's sparse attention and MoE, with its own + latent/indexer pages, followed by shared_head.norm. Unlike the main decoder, + the MTP layer has no mHC weights. + + The first draft step shares the target's packed context/verification layout. + Later steps read live metadata lengths after the speculative worker rewinds + them. The target supplies post-final-norm hidden states. """ def __init__( @@ -2459,7 +2124,7 @@ def __init__( "glm5_next MTP runs one-model speculative decoding only (MTP / MTP_EAGLE_ONE_MODEL)" ) _normalize_glm5_next_top_config(model_config.pretrained_config) - config = get_glm5_next_text_config(model_config.pretrained_config) + config = unwrap_glm5_next_text_config(model_config.pretrained_config) schedule = resolve_glm5_next_schedule(model_config.pretrained_config) num_nextn = int(getattr(config, "num_nextn_predict_layers", 0) or 0) if not (schedule.num_layers <= layer_idx < schedule.num_layers + num_nextn): @@ -2522,26 +2187,16 @@ def forward( hidden_states = torch.chunk(hidden_states, mapping.tp_size, dim=-1)[mapping.tp_rank] hidden_states = self.eh_proj(hidden_states) - runtime_ctx = build_glm5_next_runtime_context(attn_metadata, kv_lens_source="metadata") + runtime_ctx = build_glm5_next_runtime_context(attn_metadata) residual = hidden_states attn_in = self.input_layernorm(hidden_states) - parts = [] - if runtime_ctx.num_contexts > 0: - parts.append( - self.self_attn.forward_prefill( - attn_in[: runtime_ctx.num_ctx_tokens], - **runtime_ctx.sparse_kwargs(self.layer_idx, "prefill"), - ) - ) - if runtime_ctx.num_generations > 0: - phase = runtime_ctx.gen_phase - parts.append( - getattr(self.self_attn, f"forward_{phase}")( - attn_in[runtime_ctx.num_ctx_tokens :], - **runtime_ctx.sparse_kwargs(self.layer_idx, phase), - ) + if runtime_ctx.num_contexts > 0 and runtime_ctx.num_generations > 0: + attn_out = self.self_attn.forward_mixed(attn_in, **runtime_ctx.mixed_kwargs()) + else: + phase = "prefill" if runtime_ctx.num_contexts > 0 else runtime_ctx.gen_phase + attn_out = getattr(self.self_attn, f"forward_{phase}")( + attn_in, **runtime_ctx.sparse_kwargs(phase) ) - attn_out = parts[0] if len(parts) == 1 else torch.cat(parts, dim=0) hidden_states = residual + attn_out residual = hidden_states diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py index 2f373f25606d..d08ca110ff2f 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py @@ -1,50 +1,15 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -"""GLM-5.3-Flash vision tower and multimodal wrapper (``glm5_next``). - -The checkpoint publishes ``Glm5NextForConditionalGeneration``: the accepted -text decoder (:class:`~.modeling_glm5_next.Glm5NextForCausalLM`, hybrid KDA + -sparse-MLA, FP8) plus a 24-block BF16 vision tower. This module adds the -vision side directly on TensorRT-LLM's existing ``Attention`` abstraction — -TRTLLM backend, ``kv_cache_manager=None``, module-side per-head Q/K RMSNorm -and two-axis (h, w) rotary embedding — and composes a thin -``MultimodalModelMixin`` wrapper around the unchanged text decoder. There is -no pure-PyTorch/SDPA production attention path. - -Source semantics (HF ``Glm5NextVisionModel``, the named reference checkout): - -* patch embed: ``Conv3d(3, 1024, kernel=stride=(2, 14, 14))`` over packed - ``(N, 1176)`` patch rows (``3*2*14*14``), emitted by the checkpoint's - ``Glm5NextImageProcessor`` in spatial-merge-block order; -* 24 pre-norm blocks: RMSNorm → biased fused QKV (16 heads × 64) → per-head - RMSNorm on Q and K (fp32 math, eps ``rms_norm_eps``) → 2-axis rotary over - the full head dim in fp32 (16 freqs per axis, theta 10000, neox pairing) - → full non-causal attention within each image segment → biased output - projection; RMSNorm → biased clamped-SwiGLU MLP (limit ``swiglu_limit``); -* post RMSNorm → 2×2 ``Conv2d`` downsample (1024 → 4096) over each - spatial-merge block → projector ``linear → LayerNorm → GELU → clamped - SwiGLU → linear`` (all projector linears bias-free), one 4096-wide row per - image token. - -Per-image full attention reuses the Qwen-VL vision metadata contract -(:func:`~.modeling_qwen2vl._prepare_qwen_vl_vision_attn_metadata`): every -image is one context segment with ``PredefinedAttentionMask.FULL`` and no KV -cache. TP4 shards QKV / gate&up column-wise and o_proj / down row-wise; -norms, convolutions and the projector's first linear stay replicated. - -The prompt contract matches the sealed native-HF MMMU reference: OpenAI-style -content parts with every ```` placeholder interleaved at its original -position (``interleave_placeholders=True``, ``ContentFormat.OPENAI``); the -chat template itself expands each image part to -``<|begin_of_image|><|image|><|end_of_image|>`` and the input processor -expands ``<|image|>`` to ``grid.prod() / spatial_merge_size**2`` image tokens. - -Image and video preprocessing come from the Hugging Face ``Glm5NextProcessor`` -(``AutoProcessor``), exactly as Qwen2-VL and the other VLMs in this tree do; -this uses the GLM-specific Transformers revision documented in the deployment guide. - -Images and videos share the tower. The Hugging Face processor supplies the -timestamped frame structure for videos. +"""GLM-5.3-Flash vision encoder and conditional-generation wrapper. + +Images and video frame pairs share a BF16 tower: patch embedding, pre-norm +blocks, post-norm, spatial downsampling and a projector into text embeddings. +Blocks use per-head Q/K RMSNorm, FP32 two-axis RoPE and clamped SwiGLU. + +Vision attention uses the TRTLLM backend with per-item full-attention segments +and no KV cache, reusing Qwen-VL metadata preparation. The HF Glm5NextProcessor +handles preprocessing and placeholder expansion; the required Transformers +revision is documented in the deployment guide. """ import copy @@ -82,6 +47,7 @@ from ..modules.linear import Linear, TensorParallelMode, WeightMode, WeightsLoadingConfig from ..modules.rms_norm import RMSNorm from ..modules.swiglu import swiglu +from ..pyexecutor.config_utils import unwrap_glm5_next_text_config from ..utils import torch_compiling if IS_FLASHINFER_AVAILABLE: @@ -89,11 +55,7 @@ else: # pragma: no cover - CPU / no-flashinfer environments flashinfer_apply_rope_with_cos_sin_cache_inplace = None from .modeling_auto import AutoModelForCausalLM -from .modeling_glm5_next import ( - Glm5NextForCausalLM, - get_glm5_next_text_config, - glm5_next_attention_mapping, -) +from .modeling_glm5_next import Glm5NextForCausalLM, glm5_next_attention_mapping from .modeling_multimodal_encoder import MultimodalEncoderMixin from .modeling_multimodal_mixin import ( EncoderGroup, @@ -138,7 +100,7 @@ def _image_encoder_cuda_graph_config( def _text_dtype(model_config: ModelConfig[PretrainedConfig]) -> torch.dtype: - text_config = get_glm5_next_text_config(model_config.pretrained_config) + text_config = unwrap_glm5_next_text_config(model_config.pretrained_config) dtype = getattr(text_config, "dtype", None) if isinstance(dtype, str): dtype = getattr(torch, dtype) @@ -146,16 +108,7 @@ def _text_dtype(model_config: ModelConfig[PretrainedConfig]) -> torch.dtype: def _require_trtllm_vision_backend(model_config: ModelConfig[PretrainedConfig], where: str) -> None: - """Fail closed on any attention backend other than plain TRTLLM. - - The vision bring-up contract admits exactly one production attention - path: the TRTLLM backend with full-mask per-image segments and no KV - cache. VANILLA/FlashInfer (and the generic sparse-attention wrappers, - which swap in a different kernel/metadata contract) must not be - constructible for the vision tower — a configured non-TRTLLM backend is - an error here, never a fallback. The text decoder keeps its own - independently validated configuration. - """ + """Require full-mask TRTLLM vision attention without sparse backend wrappers.""" backend = getattr(model_config, "attn_backend", None) if not isinstance(backend, str) or backend.upper() != "TRTLLM": raise ValueError( @@ -193,7 +146,7 @@ class Glm5NextVisionAttention(Attention): def __init__(self, model_config: ModelConfig[PretrainedConfig], layer_idx: int) -> None: _require_trtllm_vision_backend(model_config, type(self).__name__) config = model_config.pretrained_config.vision_config - text_config = get_glm5_next_text_config(model_config.pretrained_config) + text_config = unwrap_glm5_next_text_config(model_config.pretrained_config) dtype = _text_dtype(model_config) super().__init__( hidden_size=config.hidden_size, @@ -731,14 +684,8 @@ def __init__( # ``modules_to_not_convert``); scrub the quant config so no Linear # picks up FP8 behavior. self.model_config.quant_config = QuantConfig() - # Every TP collective this model constructs is pinned to NCCL: the - # text decoder pins all of its reductions (AUTO's runtime-selected - # tactic raced at decode on TP4), and the tower's row-parallel - # projections must stay on the same deterministic contract so image - # requests cannot reintroduce the unpinned path. The engine hands the - # wrapper a frozen ModelConfig (and the wrapper hands this class its - # own deepcopy), so use the documented `_frozen` bypass and restore - # the incoming state. + # Pin vision TP reductions to NCCL to avoid runtime autotuning. + # This wrapper owns a config copy; preserve its incoming frozen state. from ..distributed import AllReduceStrategy was_frozen = self.model_config._frozen @@ -840,14 +787,9 @@ def _glm5_next_build_batched_input(multimodal_params: List[MultimodalParams]) -> class Glm5NextInputProcessor(BaseMultimodalInputProcessor, BaseMultimodalDummyInputsBuilder): - """Input processor on the Hugging Face ``Glm5NextProcessor`` (image and - video processors + placeholder expansion), the Qwen2-VL pattern. - - Uses the GLM-specific Transformers revision documented in the deployment guide. The - processor performs smart resize, - normalization, patchification, frame sampling / timestamped video token - layout and the ``<|image|>`` / ``<|video|>`` expansion; this class only - routes the outputs into the engine's payload format. + """Adapt HF image/video preprocessing and placeholder expansion to engine inputs. + + Requires the GLM-specific Transformers revision in the deployment guide. """ def __init__( @@ -865,7 +807,7 @@ def __init__( trust_remote_code=trust_remote_code, **kwargs, ) - text_config = get_glm5_next_text_config(config) + text_config = unwrap_glm5_next_text_config(config) dtype = getattr(text_config, "dtype", None) or torch.bfloat16 self._dtype = getattr(torch, dtype) if isinstance(dtype, str) else dtype self._tokenizer = ( @@ -899,7 +841,7 @@ def dtype(self) -> torch.dtype: return self._dtype def get_vocab_size(self) -> int: - return int(get_glm5_next_text_config(self._config).vocab_size) + return int(unwrap_glm5_next_text_config(self._config).vocab_size) def get_preferred_media_io_kwargs(self) -> Dict[str, Dict[str, Any]]: # PIL is the HF processor's native input; the server's default float @@ -1082,10 +1024,7 @@ def call_with_text_prompt( Glm5NextInputProcessor, model_type="glm5_next", placeholder_metadata=MultimodalPlaceholderMetadata( - # The chat template consumes OpenAI-style content parts natively and - # emits `<|begin_of_image|><|image|><|end_of_image|>` per image part; - # interleaving preserves each image's original position in the text — - # the contract the sealed native-HF MMMU reference recorded. + # Preserve each image/video placeholder at its original prompt position. placeholder_map={ "image": "<|begin_of_image|><|image|><|end_of_image|>", "video": "<|begin_of_video|><|video|><|end_of_video|>", @@ -1096,14 +1035,10 @@ def call_with_text_prompt( ), ) class Glm5NextVLM(MultimodalModelMixin, PreTrainedModel): - """Thin conditional-generation wrapper around the accepted text decoder. - - The inner LM is the unchanged :class:`Glm5NextForCausalLM` (constructed - through its own registered architecture name), so text-only requests run - the sealed text path byte-for-byte — the vision processor/tower is only - reachable when a request carries image or video data. The text decoder keeps - ``KVCacheManagerV2`` ownership; the vision tower runs context-only with - no cache manager. + """Compose the text decoder with an optional image/video encoder. + + The inner decoder owns recurrent/KV state and speculative decoding. Text-only + requests bypass the vision tower, which runs full attention without KV cache. """ _supports_flash_attn = True diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index 35ad04c9e5b4..e5a47ac9a497 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -274,8 +274,9 @@ def get_kv_cache_manager_cls( "indexer state is a V2 extra buffer). Leave " "kv_cache_config.use_kv_cache_manager_v2='auto' or set it " "to True.") - from ..models.modeling_glm5_next import glm5_next_cache_manager_cls - return glm5_next_cache_manager_cls() + from ..attention.backends.sparse.glm_kpool import \ + Glm5NextCacheManager + return Glm5NextCacheManager # Kimi K3 (KDA + MLA hybrid): block reuse uses the unified C++ pool # (CppMambaHybridCacheManager) like the other hybrid linear models — diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index 175a2df2da80..d5ebfd1e9aae 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -2273,34 +2273,29 @@ def test_fp8_nixl(self, mtp): "decoding_type": "MTP", "max_draft_len": 3, } if mtp else None - ctx_server_config = { + common_config = { "tensor_parallel_size": 4, "pipeline_parallel_size": 1, "moe_expert_parallel_size": 4, "max_batch_size": 64, "max_num_tokens": 16384, "max_seq_len": 8192, - "disable_overlap_scheduler": True, - "cuda_graph_config": None, "kv_cache_config": kv_cache_config, "speculative_config": speculative_config, "cache_transceiver_config": cache_transceiver_config, } + ctx_server_config = { + **common_config, + "disable_overlap_scheduler": True, + "cuda_graph_config": None, + } gen_server_config = { - "tensor_parallel_size": 4, - "pipeline_parallel_size": 1, - "moe_expert_parallel_size": 4, - "max_batch_size": 64, - "max_num_tokens": 16384, - "max_seq_len": 8192, + **common_config, "disable_overlap_scheduler": False, "cuda_graph_config": { "max_batch_size": 64, "enable_padding": True, }, - "kv_cache_config": kv_cache_config, - "speculative_config": speculative_config, - "cache_transceiver_config": cache_transceiver_config, } disaggregated_server_config = { "hostname": "localhost", diff --git a/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py b/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py index 400923f2c556..505d186d042c 100644 --- a/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py +++ b/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py @@ -25,12 +25,10 @@ AttentionInputType, ) from tensorrt_llm._torch.attention.backends.sparse.glm_kpool import ( - INDEX_SENTINEL, GlmKpoolSparseAttention, GlmKpoolSparseParams, latent_pool_rows, paged_slot_indices, - positions_to_pool_rows, ) from tensorrt_llm._torch.attention.backends.sparse.glm_kpool.params import ( GlmKpoolBackendForwardArgs, @@ -61,23 +59,16 @@ def forward_case(): ) -@pytest.mark.parametrize( - "route", ["context", "generation", "verify", "paged_context", "paged_generation"] -) +@pytest.mark.parametrize("route", ["context", "generation", "verify"]) @pytest.mark.parametrize("supply_output", [False, True]) def test_forward_dispatches_latent_rows_and_preserves_output(forward_case, route, supply_output): case = forward_case - context = route in ("context", "paged_context") + context = route == "context" q = case.q.repeat_interleave(2, dim=0) if route == "verify" else case.q indices = case.indices.repeat_interleave(2, dim=0) if route == "verify" else case.indices core_output = torch.arange(q.shape[0] * 8, dtype=q.dtype).view(q.shape[0], 2, 4) case.backend._dispatch_sparse_core.return_value = core_output - latent_prefix = torch.zeros(3, 4, dtype=q.dtype) if route == "context" else None - selection = ( - GlmKpoolBackendForwardArgs(topk_rows=indices) - if route.startswith("paged_") - else GlmKpoolBackendForwardArgs(topk_indices=indices) - ) + selection = GlmKpoolBackendForwardArgs(topk_rows=indices) output = torch.empty(q.shape[0], 8, dtype=q.dtype) if supply_output else None args = AttentionForwardArgs( attention_input_type=( @@ -86,7 +77,7 @@ def test_forward_dispatches_latent_rows_and_preserves_output(forward_case, route sparse_backend_args=selection, output=output, ) - actual = case.backend.forward(q, latent_prefix, None, case.metadata, args) + actual = case.backend.forward(q, None, None, case.metadata, args) torch.testing.assert_close(actual, core_output.flatten(1)) if supply_output: assert actual is output @@ -96,36 +87,38 @@ def test_forward_dispatches_latent_rows_and_preserves_output(forward_case, route case.backend._dispatch_sparse_core.call_args.args ) torch.testing.assert_close(dispatched_q, q.view(-1, 2, 4)) - if route == "context": - torch.testing.assert_close(dispatched_k, latent_prefix.view(3, 1, 4)) - torch.testing.assert_close(dispatched_indices, indices) - else: - torch.testing.assert_close(dispatched_k, case.state.latent_pool.view(-1, 1, 4)) - expected = ( - indices - if route.startswith("paged_") - else torch.tensor([[8, -1], [1, 2]], dtype=torch.int32) - ) - if route == "verify": - expected = expected.repeat_interleave(2, dim=0) - torch.testing.assert_close(dispatched_indices, expected) + torch.testing.assert_close(dispatched_k, case.state.latent_pool.view(-1, 1, 4)) + torch.testing.assert_close(dispatched_indices, indices) @pytest.mark.parametrize( "invalid", - ["selection", "v", "out_scale", "out_scale_sf", "output_sf", "shape", "dtype", "device"], + [ + "selection", + "legacy_indices", + "v", + "out_scale", + "out_scale_sf", + "output_sf", + "shape", + "dtype", + "device", + ], ) def test_forward_rejects_invalid_arguments_before_cache_access(forward_case, invalid): case = forward_case args = AttentionForwardArgs( attention_input_type=AttentionInputType.context_only, - sparse_backend_args=GlmKpoolBackendForwardArgs(topk_indices=case.indices), + sparse_backend_args=GlmKpoolBackendForwardArgs(topk_rows=case.indices), ) v = None error, message = ValueError, "quantized attention output" if invalid == "selection": args.sparse_backend_args = None - error, message = NotImplementedError, "pool-expanded selection" + error, message = ValueError, "pool-expanded selection" + elif invalid == "legacy_indices": + args.sparse_backend_args.topk_indices = case.indices + message = "not request-local topk_indices" elif invalid == "v": v = torch.zeros(1) message = "v must be None" @@ -145,14 +138,9 @@ def test_forward_rejects_invalid_arguments_before_cache_access(forward_case, inv case.backend._dispatch_sparse_core.assert_not_called() -@pytest.mark.parametrize("paged", [False, True]) -def test_forward_rejects_mixed_phase(forward_case, paged): +def test_forward_rejects_mixed_phase(forward_case): case = forward_case - selection = ( - GlmKpoolBackendForwardArgs(topk_rows=case.indices) - if paged - else GlmKpoolBackendForwardArgs(topk_indices=case.indices) - ) + selection = GlmKpoolBackendForwardArgs(topk_rows=case.indices) args = AttentionForwardArgs( attention_input_type=AttentionInputType.mixed, sparse_backend_args=selection ) @@ -162,29 +150,29 @@ def test_forward_rejects_mixed_phase(forward_case, paged): @pytest.mark.parametrize("context", [False, True]) -def test_forward_requires_phase_specific_latent_source(forward_case, context): +def test_forward_rejects_explicit_latent_source(forward_case, context): case = forward_case args = AttentionForwardArgs( attention_input_type=( AttentionInputType.context_only if context else AttentionInputType.generation_only ), - sparse_backend_args=GlmKpoolBackendForwardArgs(topk_indices=case.indices), + sparse_backend_args=GlmKpoolBackendForwardArgs(topk_rows=case.indices), ) - k = None if context else torch.zeros(3, 4, dtype=case.q.dtype) - message = "contiguous latent prefix as k" if context else "k must be None" + k = torch.zeros(3, 4, dtype=case.q.dtype) + message = "k must be None" with pytest.raises(ValueError, match=message): case.backend.forward(case.q, k, None, case.metadata, args) case.backend._dispatch_sparse_core.assert_not_called() -def test_cache_state_prefers_live_metadata_and_rejects_graph_fallback(): +@pytest.mark.parametrize("is_cuda_graph", [False, True]) +def test_cache_state_uses_prepared_metadata_without_fallback(is_cuda_graph): backend = object.__new__(GlmKpoolSparseAttention) backend.layer_idx = 0 latent = torch.zeros(4, 8, 1, 512, dtype=torch.bfloat16) index = torch.zeros(4, 8, 1, 384, dtype=torch.bfloat16) tables = torch.tensor([[2, 0], [3, 1]], dtype=torch.long) live_lengths = torch.tensor([5, 8], dtype=torch.int32) - stale_lengths = torch.tensor([9, 12], dtype=torch.long) metadata = SimpleNamespace( kv_cache_manager=SimpleNamespace( tokens_per_block=8, @@ -193,9 +181,9 @@ def test_cache_state_prefers_live_metadata_and_rejects_graph_fallback(): ), seq_lens=torch.ones(2, dtype=torch.long), num_contexts=0, - is_cuda_graph=True, + is_cuda_graph=is_cuda_graph, kv_lens_cuda=live_lengths, - mamba_metadata=SimpleNamespace(glm_block_tables=tables, glm_kv_lens=stale_lengths), + mamba_metadata=SimpleNamespace(glm_block_tables=tables), ) state = backend._cache_state(metadata) assert state.block_tables.data_ptr() == tables.data_ptr() @@ -203,8 +191,12 @@ def test_cache_state_prefers_live_metadata_and_rejects_graph_fallback(): torch.testing.assert_close(state.kv_lens, live_lengths) metadata.mamba_metadata.glm_block_tables = None with patch("torch.zeros", side_effect=AssertionError("must not allocate fallback tables")): - with pytest.raises(RuntimeError, match="CUDA-graph execution requires"): + with pytest.raises(RuntimeError, match="requires prepared glm_block_tables"): backend._cache_state(metadata) + metadata.mamba_metadata.glm_block_tables = tables + metadata.kv_lens_cuda = None + with pytest.raises(ValueError, match="kv_lens_cuda"): + backend._cache_state(metadata) @pytest.mark.parametrize("heads", [16, 64]) @@ -257,15 +249,6 @@ def test_paged_slot_indices_returns_page_and_offset_pairs(): assert torch.equal(offset, torch.tensor([[0, 5, 0], [3, 0, 7]])) -def test_positions_to_pool_rows_preserves_sentinels(): - table = torch.tensor([[7, 2], [4, 0]]) - positions = torch.tensor([[0, 9, INDEX_SENTINEL], [15, INDEX_SENTINEL, 8]], dtype=torch.int32) - rows = positions_to_pool_rows(positions, table, 8, base_row=3, rows_per_slot=10) - assert rows.dtype == torch.int32 - # base + slot * rows_per_slot + within_page; sentinels stay -1, never clamped. - assert torch.equal(rows, torch.tensor([[73, 24, -1], [10, -1, 3]], dtype=torch.int32)) - - def test_latent_pool_rows_reinterprets_a_coalesced_pool_without_copying(): dim, tpb, slots = 16, 4, 3 # A wider shared storage: each slot holds this buffer's page plus another diff --git a/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py b/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py index 8e7aac05ca10..441044ff1fd4 100644 --- a/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py +++ b/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py @@ -322,11 +322,7 @@ def reference(): @pytest.mark.parametrize("context", [False, True]) def test_fp8_cache_storage_helpers_preserve_coalesced_pages(context): - """Check cache-write and compatibility prefix-gather helpers. - - The production model supplies topk_rows and does not call gather_paged_prefix; - its FP8 attention core is covered by the sparse-core test above. - """ + """Write FP8 payloads without disturbing coalesced pages or indexer pool keys.""" from types import SimpleNamespace from unittest.mock import Mock @@ -351,11 +347,16 @@ def test_fp8_cache_storage_helpers_preserve_coalesced_pages(context): packed = torch.tensor([[1.0, 2.0, 3.0, 4.0]], device="cuda") positions = torch.tensor([0] if context else [[0]], device="cuda") backend.append_paged_state( - latent, packed, positions, object(), request_index=0 if context else None + latent, + packed, + positions, + object(), + request_ids=torch.zeros(1, dtype=torch.int32, device="cuda") if context else None, ) - decoded, actual_packed = backend.gather_paged_prefix(1, object(), request_index=0) + decoded = latent_pool[2, 0].float() * backend.kv_scale_quant_orig + actual_packed = index_pool[2, 0, :4] expected = (latent * 2).clamp(-448, 448).to(torch.float8_e4m3fn).float() * 0.5 - torch.testing.assert_close(decoded.float(), expected) - torch.testing.assert_close(actual_packed.float(), packed) + torch.testing.assert_close(decoded.unsqueeze(0), expected) + torch.testing.assert_close(actual_packed.float().unsqueeze(0), packed) assert torch.count_nonzero(storage.float()[[0, 2, 3, 5, 6, 8]]) == 0 assert torch.count_nonzero(index_pool[:, :, 4:]) == 0 diff --git a/tests/unittest/_torch/modeling/test_glm5_next_contracts.py b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py index 24dacea071e8..c71170b9df88 100644 --- a/tests/unittest/_torch/modeling/test_glm5_next_contracts.py +++ b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py @@ -19,8 +19,10 @@ SPARSE_MLP, Glm5NextDecoderLayer, Glm5NextLinearAttention, + Glm5NextMTP, Glm5NextRuntimeContext, Glm5NextSparseAttention, + build_glm5_next_runtime_context, glm5_next_tp_reduces, ) from tensorrt_llm._torch.models.modeling_glm5_next_vision import Glm5NextVisionModelBase @@ -95,6 +97,45 @@ def test_layer_masks_accept_composite_and_text_configs(): get_glm5_next_layer_masks(config) +@pytest.mark.cpu_only +@pytest.mark.parametrize("is_cuda_graph", [False, True]) +def test_runtime_context_uses_prepared_schedules_and_live_lengths(is_cuda_graph): + live_lengths = torch.tensor([6, 9], dtype=torch.int32) + prepared = SimpleNamespace( + glm_block_tables=torch.tensor([[0, 1], [2, 3]]), + glm_ctx_cu_seqlens=[0, 3], + glm_cached_lens_host=[3, 5], + ) + metadata = SimpleNamespace( + kv_cache_manager=object(), + mamba_metadata=prepared, + seq_lens=torch.tensor([3, 4]), + num_contexts=1, + num_ctx_tokens=3, + num_tokens=7, + kv_lens_cuda=live_lengths, + is_cuda_graph=is_cuda_graph, + ) + context = build_glm5_next_runtime_context(metadata) + assert context.ctx_cu_seqlens is prepared.glm_ctx_cu_seqlens + assert context.cached_lens is prepared.glm_cached_lens_host + assert context.gen_phase == "verify" + assert context.gen_tokens_per_request == 4 + torch.testing.assert_close(context.kv_lens, live_lengths.long()) + # MTP rewinds the device lengths without changing the host prefill schedule. + live_lengths[1] -= 2 + torch.testing.assert_close( + build_glm5_next_runtime_context(metadata).kv_lens, torch.tensor([6, 7]) + ) + prepared.glm_block_tables = None + with pytest.raises(RuntimeError, match="requires prepared glm_block_tables"): + build_glm5_next_runtime_context(metadata) + prepared.glm_block_tables = torch.tensor([[0, 1], [2, 3]]) + metadata.kv_lens_cuda = None + with pytest.raises(ValueError, match="kv_lens_cuda"): + build_glm5_next_runtime_context(metadata) + + @pytest.mark.cpu_only def test_checkpoint_routes_vision_and_optional_mtp_separately(): config = _config() @@ -367,27 +408,29 @@ def run(lengths, discard_prefix=False): @pytest.mark.cpu_only @pytest.mark.parametrize("tokens_per_request", [1, 4], ids=["decode", "verify"]) -def test_attention_dp_mixed_batches_keep_one_full_batch_moe_call(tokens_per_request: int) -> None: - # Rank-local phase mixes differ under ADP. Exercise the real decoder - # dispatch and FFN wrapper with lightweight math stubs. +@pytest.mark.parametrize("attention_dp", [False, True], ids=["tp", "attention-dp"]) +@pytest.mark.parametrize("layer_kind", ["decoder", "mtp"]) +def test_mixed_batches_keep_one_full_batch_moe_call( + tokens_per_request: int, attention_dp: bool, layer_kind: str +) -> None: + # ADP ranks may have different phase mixes, but must issue one MoE call + # per layer. TP attention must reduce its output once for the full batch. batches = [(3, 1), (3, 0), (0, 1), (2, 1)] counts = [ context_tokens + generations * tokens_per_request for context_tokens, generations in batches ] for rank, (context_tokens, generations) in enumerate(batches): mapping = Mapping( - world_size=4, tp_size=4, moe_ep_size=4, rank=rank, enable_attention_dp=True + world_size=4, tp_size=4, moe_ep_size=4, rank=rank, enable_attention_dp=attention_dp ) contexts = int(context_tokens > 0) metadata = SimpleNamespace(all_rank_num_tokens=counts) context = Glm5NextRuntimeContext( - manager=object(), num_contexts=contexts, num_ctx_tokens=context_tokens, num_generations=generations, ctx_cu_seqlens=[0, context_tokens] if contexts else [0], cached_lens=[0] * contexts + [1] * generations, - state_indices=torch.arange(contexts + generations), kv_lens=torch.tensor( ([context_tokens] if contexts else []) + ([1 + tokens_per_request] if generations else []) @@ -400,6 +443,10 @@ def test_attention_dp_mixed_batches_keep_one_full_batch_moe_call(tokens_per_requ attention = SimpleNamespace( tp_all_reduce=reduction if glm5_next_tp_reduces(mapping) else None ) + + def attend(x, *args, reduce=True, **kwargs): + return attention.tp_all_reduce(x) if reduce and attention.tp_all_reduce else x + for phase in ("prefill", "decode", "verify"): implementation = MethodType( getattr(Glm5NextSparseAttention, f"forward_{phase}"), attention @@ -407,7 +454,7 @@ def test_attention_dp_mixed_batches_keep_one_full_batch_moe_call(tokens_per_requ setattr( attention, f"forward_{phase}", - create_autospec(implementation, side_effect=lambda x, *args, **kwargs: x), + create_autospec(implementation, side_effect=attend), ) attention.forward_mixed = MethodType(Glm5NextSparseAttention.forward_mixed, attention) connection = SimpleNamespace( @@ -425,11 +472,37 @@ def test_attention_dp_mixed_batches_keep_one_full_batch_moe_call(tokens_per_requ post_attention_layernorm=torch.nn.Identity(), mlp=Mock(side_effect=lambda x, all_rank_num_tokens: x), ) - layer.forward_direct = MethodType(Glm5NextDecoderLayer.forward_direct, layer) - layer.run_mlp = MethodType(Glm5NextDecoderLayer.run_mlp, layer) - result = Glm5NextDecoderLayer.forward(layer, hidden_states=hidden, runtime_ctx=context) - assert result.shape == hidden.shape - reduction.assert_not_called() + if layer_kind == "decoder": + attn_input = hidden.mean(dim=1) + layer.forward_direct = MethodType(Glm5NextDecoderLayer.forward_direct, layer) + layer.run_mlp = MethodType(Glm5NextDecoderLayer.run_mlp, layer) + result = Glm5NextDecoderLayer.forward(layer, hidden_states=hidden, runtime_ctx=context) + expected = hidden + attn_input.unsqueeze(1) + expected = expected + expected.mean(dim=1, keepdim=True) + else: + hidden = hidden.mean(dim=1) + input_ids = torch.arange(hidden.shape[0]) + embed_tokens = torch.nn.Embedding(hidden.shape[0], 8) + layer.enorm = layer.hnorm = torch.nn.Identity() + layer.model_config = SimpleNamespace(mapping=mapping) + layer.eh_proj = torch.nn.Linear(16 if attention_dp else 4, 8, bias=False) + layer.shared_head = SimpleNamespace(norm=torch.nn.Identity()) + projected_input = torch.cat([embed_tokens(input_ids), hidden], dim=-1) + if not attention_dp: + projected_input = projected_input.chunk(mapping.tp_size, dim=-1)[mapping.tp_rank] + attn_input = layer.eh_proj(projected_input) + with patch( + "tensorrt_llm._torch.models.modeling_glm5_next.build_glm5_next_runtime_context", + return_value=context, + ): + result = Glm5NextMTP.forward(layer, input_ids, None, hidden, embed_tokens, metadata) + expected = 4 * attn_input + torch.testing.assert_close(result, expected) + if attention_dp: + reduction.assert_not_called() + else: + reduction.assert_called_once() + torch.testing.assert_close(reduction.call_args.args[0], attn_input) layer.mlp.assert_called_once() mlp_tokens, all_rank_num_tokens = layer.mlp.call_args.args assert mlp_tokens.shape[0] == counts[rank] @@ -437,7 +510,7 @@ def test_attention_dp_mixed_batches_keep_one_full_batch_moe_call(tokens_per_requ if contexts: attention.forward_prefill.assert_called_once() torch.testing.assert_close( - attention.forward_prefill.call_args.args[0], hidden.mean(dim=1)[:context_tokens] + attention.forward_prefill.call_args.args[0], attn_input[:context_tokens] ) else: attention.forward_prefill.assert_not_called() @@ -445,9 +518,7 @@ def test_attention_dp_mixed_batches_keep_one_full_batch_moe_call(tokens_per_requ if generations: generation.assert_called_once() assert generation.call_args.kwargs["metadata"] is metadata - torch.testing.assert_close( - generation.call_args.args[0], hidden.mean(dim=1)[context_tokens:] - ) + torch.testing.assert_close(generation.call_args.args[0], attn_input[context_tokens:]) if contexts: assert generation.call_args.kwargs["reduce"] is False if tokens_per_request > 1: From 6e448150371880a21da3fb55a73f7e064ef21557 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Wed, 16 Sep 2026 05:34:49 -0700 Subject: [PATCH 08/35] [None][refactor] Trim GLM-5.3-Flash model helpers and comments Remove unused query and diagnostic wrappers and redundant geometry fields. Rename the shared attention/indexer projection helper to reflect its use across prefill, decode and verify, and shorten model and vision comments. Preserve computation and dispatch behavior. Validated with 35 focused model tests, applicable pre-commit hooks, and normalized AST comparison. Signed-off-by: Ruocheng Jia --- .../_torch/models/modeling_glm5_next.py | 205 ++++-------------- .../models/modeling_glm5_next_vision.py | 28 +-- 2 files changed, 50 insertions(+), 183 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next.py b/tensorrt_llm/_torch/models/modeling_glm5_next.py index 003c8f8de4b2..2916aa372e18 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next.py @@ -23,7 +23,7 @@ import os import re -from collections.abc import Iterable, Sequence +from collections.abc import Sequence from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any, Callable @@ -57,7 +57,6 @@ from .checkpoints.hf.glm5_next_weight_mapper import ( Disposition, Glm5NextHfWeightMapper, - Glm5NextWeightAudit, glm5_next_is_quantized, ) from .modeling_deepseekv3 import DeepseekV3Gate @@ -543,20 +542,6 @@ def get_preferred_kv_cache_manager_version(cls, pretrained_config=None) -> str: """Use V2 for the shared latent-KV, pool-indexer and recurrent-state cache.""" return "V2" - def attention_type(self, layer_idx: int) -> str: - """The literal attention module type for ``layer_idx``.""" - return self.schedule.attention[layer_idx] - - def mlp_type(self, layer_idx: int) -> str: - """The literal feed-forward module type for ``layer_idx``.""" - return self.schedule.mlp[layer_idx] - - def audit_checkpoint(self, keys: Iterable[str]) -> Glm5NextWeightAudit: - """Resolve every checkpoint key against this model's destinations.""" - mapper = Glm5NextHfWeightMapper() - mapper.init_model_and_config(self, self.model_config) - return mapper.audit(keys) - # -- whole-model materialization -------------------------------------- def load_weights( @@ -812,23 +797,11 @@ def mark_linear(mod_path: str) -> None: class Glm5NextLinearAttention(KimiKDALinearAttention): - """GLM-5.3-Flash KDA layer: the shared Kimi KDA mixer, configured. - - The recurrence, convolution, projections, mixed context+generation - batches and the speculative-verify replay kernel are all the shared - module's (``trtllm::kda_prefill`` / ``kda_decode`` / ``kda_mtp_decode`` - with the FLA fallbacks), reading the ``MambaHybridCacheManagerV2`` pools - the way Kimi K3 does. What GLM-5.3-Flash configures differently: - - * the **low-rank output gate** ``g_b_proj(g_a_proj(x))`` (the checkpoint - has no full-rank ``g_proj``), selected through the mixer's own - ``use_full_rank_gate`` config key and served by its fused - ``[f_a | g_a | b]`` / ``[f_b; g_b]`` decode projections; - * ``A_log`` / ``dt_bias`` are **published in fp32** and kept so. - - The exact-placement loader shards this rank's head range itself - (:meth:`shard_checkpoint_tensor`), as the projections are the mixer's own - local-width ``nn.Linear`` modules rather than Mapping-aware ``Linear``. + """Configure the shared KDA mixer for GLM's low-rank output gate and FP32 gates. + + Recurrence, convolution and speculative replay use shared KDA kernels. + The loader shards local-width projections by head range; A_log and dt_bias + retain the checkpoint's FP32 precision. """ #: Checkpoint tensors sharded by this rank's head *channel* range on dim 0. @@ -869,10 +842,8 @@ def __init__( super().__init__( config, layer_idx, mapping=mapping, allreduce_strategy=glm5_next_allreduce_strategy() ) - # The checkpoint publishes ``A_log`` / ``dt_bias`` in fp32 (the mixer - # stores them in bf16 by default; the kernels consume fp32 copies - # either way). Rebuilt without ``.detach``/``.data``, which MetaInitMode - # rejects on meta tensors. + # Preserve checkpoint FP32 gate parameters. MetaInitMode rejects detach + # on meta tensors, so create empty FP32 parameters on that branch. for name in ("A_log", "dt_bias"): param = getattr(self, name) data = ( @@ -884,8 +855,6 @@ def __init__( self.tp_size = self._kda_tp_size self.tp_rank = self._kda_tp_rank self.total_num_heads = int(linear["num_heads"]) - self.qkv_dim = self.proj_size - self.total_qkv_dim = self.total_num_heads * self.head_dim # -- tensor-parallel ownership (consumed by the exact-placement loader) -- @@ -925,17 +894,10 @@ def finalize_weights(self) -> None: def _glm5_linear_kwargs( model_config: ModelConfig | None, mapping: Mapping | None ) -> dict[str, Any]: - """Construction arguments shared by every Mapping-aware ``Linear`` here. - - The quant config is the checkpoint's (``FP8_BLOCK_SCALES`` with its - published ``modules_to_not_convert``); the base class's - ``apply_quant_config_exclude_modules`` then flips the excluded modules to - bf16 by *runtime module name* before weights are created, so no - model-specific plan is needed. ``disable_deep_gemm`` pins the block-FP8 - GEMM to ``fp8_quantize_1x128`` + the CuTe DSL Blackwell kernel (the same - arithmetic the routed experts run; DeepGEMM is not available here). - Direct (harness) construction without a ``model_config`` yields plain - bf16 modules over ``mapping``. + """Build BF16 or block-FP8 Linear arguments with attention TP/DP mapping. + + The base model applies quantization exclusions by runtime module name. + Block-FP8 projections use the CuTe DSL GEMM path (disable_deep_gemm=True). """ return { "bias": False, @@ -953,34 +915,12 @@ def _glm5_linear_kwargs( class Glm5NextIndexer(nn.Module): - """Pool-compressed DSA indexer (``Glm5NextTextIndexer``). - - Stock DeepSeek-V3.2 DSA selects ``index_topk`` individual keys. This one - selects ``index_topk / index_kpool`` *compressed pools*, expands each back - into its member positions, and always appends the incomplete tail, so the - two are not interchangeable even though both end up with ~2048 positions. - - The backend caches each token's key and compression gate, plus the pooled - key at each pool's first row. It updates the affected pool when tokens are - appended; only complete pools are scored, and the current tail is selected - separately so future tokens cannot influence a query. - - Unlike the HF module there is no left padding here: TensorRT-LLM stores - exactly one request's tokens per cache slot starting at position 0, so - ``first_key`` is always 0 and the packed validity channel HF carries for - padded batches is replaced by the request's own ``kv_len``. - - Tensor parallelism: the whole indexer is replicated (the vLLM/SGLang - ownership) -- every rank runs all 32 scoring heads on the replicated - pool-key path (``wk``, ``k_norm``, the APE and compress gate) and selects - identical pool/tail/-1 indices by identical compute, with no collective. - The two scoring GEMMs are tiny (1536->4096, 4096->32), far cheaper than - the fp32 ``[tokens, pools]`` score all-reduce the sharded-head design - needed before top-k. - - Decode and prefill score cached *pool keys* (the trailing - ``head_dim`` columns of each pool's first-member cache row, maintained by - the backend's ``update_pool_keys``) through the fused paged kernels. + """Select complete key pools and append the incomplete causal tail. + + The backend caches [key | compression gate | pooled key] per token, storing + pooled keys at each pool's first row. Replicated projections and scoring heads + produce the same selection on each TP rank without a score all-reduce. + Packed requests use visible lengths instead of HF's left-padding mask. """ #: ``wk``, the pool-compress gate and ``weights_proj`` all read the same @@ -1011,9 +951,6 @@ def __init__( f"glm5_next indexer has {self.total_n_heads} scoring heads, not " f"divisible by tp_size {self.tp_size}" ) - # Replicated scoring heads (vLLM/SGLang ownership): every rank runs - # all 32 heads and selects identical indices by identical compute, - # so no score collective is needed before top-k. self.n_heads = self.total_n_heads self.head_dim = int(config.index_head_dim) self.index_topk = int(config.index_topk) @@ -1038,12 +975,8 @@ def __init__( self.index_kpool_compress_ape = nn.Parameter( torch.zeros(self.index_kpool, self.head_dim, dtype=torch.bfloat16) ) - # Decode top-k over the fused pool scores: the DSA indexer's TopK - # module. The CuTe-DSL radix kernel is bounded by each request's - # candidate count (1-6 us here vs 20-30 us for the CUDA radix kernel - # and ~40 us for torch.topk over the capacity-wide row); identical - # selections measured. Falls back to the CUDA radix kernel where the - # CUTLASS DSL is unavailable. + # Reuse DSA's TopK, bounded by each request's candidate count. + # Prefer CuTe DSL radix selection, with CUDA radix as the fallback. from ..cute_dsl_utils import IS_CUTLASS_DSL_AVAILABLE from ..modules.top_k import TopK, TopKImplementation @@ -1059,20 +992,9 @@ def __init__( @property def cache_state_dim(self) -> int: - """Width of one indexer cache row: ``[k | gate | pool key]``. - - The trailing ``head_dim`` columns hold, on a pool's first-member row, - the pool's compressed key -- maintained incrementally by the backend - so decode scores cached pool keys instead of rebuilding every pool - from the packed state each step. - """ + """Width of a cached [key | compression gate | pooled key] row.""" return 3 * self.head_dim - @property - def packed_state_dim(self) -> int: - """Width of the cached per-token state: ``[k | gate]``.""" - return 2 * self.head_dim - def project_state(self, hidden_states: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: """``(packed [k(head_dim) | gate(head_dim)], head weights [n_heads])`` from the one fused input GEMM.""" @@ -1081,10 +1003,6 @@ def project_state(self, hidden_states: torch.Tensor) -> tuple[torch.Tensor, torc packed = torch.cat([self.k_norm(out[:, :hd]), out[:, hd : 2 * hd]], dim=-1) return packed, out[:, 2 * hd :] - def packed_state(self, hidden_states: torch.Tensor) -> torch.Tensor: - """Per-token ``[k(head_dim) | gate(head_dim)]`` written to the cache.""" - return self.project_state(hidden_states)[0] - class Glm5NextSparseAttention(nn.Module): """NoPE sparse MLA with a pool-compressed indexer. @@ -1210,7 +1128,7 @@ def project_inputs(self, hidden_states: torch.Tensor) -> tuple[torch.Tensor, tor latent = self.kv_a_layernorm(qa_kva[:, self.q_lora_rank :]) return q_resid, latent - def _decode_projections( + def _project_attention_and_indexer( self, hidden_states: torch.Tensor ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: """``(q_resid, latent, packed [k | gate], index_weights)`` -- the two @@ -1354,7 +1272,7 @@ def forward_prefill( rows = ctx_rows_fn(kpool, device) else: rows = Glm5NextContextRows.build(cu_seqlens, cached_lens, kpool, device) - q_resid, latent, packed, index_weights = self._decode_projections(hidden_states) + q_resid, latent, packed, index_weights = self._project_attention_and_indexer(hidden_states) self.attn_backend.append_paged_state( latent, packed, rows.positions, metadata, request_ids=rows.request_ids ) @@ -1399,7 +1317,7 @@ def forward_decode( batch = hidden_states.shape[0] positions = kv_lens - 1 # [B] indexer = self.indexer - q_resid, latent, packed, index_weights = self._decode_projections(hidden_states) + q_resid, latent, packed, index_weights = self._project_attention_and_indexer(hidden_states) self.attn_backend.append_paged_state( latent.unsqueeze(1), packed.unsqueeze(1), positions.unsqueeze(1), metadata ) @@ -1463,7 +1381,7 @@ def forward_verify( device = hidden_states.device steps = torch.arange(tokens_per_request, device=device) positions = (kv_lens - tokens_per_request).unsqueeze(1) + steps # [B, T] - q_resid, latent, packed, index_weights = self._decode_projections(hidden_states) + q_resid, latent, packed, index_weights = self._project_attention_and_indexer(hidden_states) self.attn_backend.append_paged_state( latent.view(batch, tokens_per_request, -1), packed.view(batch, tokens_per_request, -1), @@ -1491,18 +1409,10 @@ def forward_verify( class Glm5NextGate(DeepseekV3Gate): - """The shared DeepSeek noaux_tc gate, kept FP32 end to end. - - Same routing math and weights as :class:`DeepseekV3Gate` (selection on - bias-corrected sigmoid scores, weights gathered from the uncorrected - scores, normalization before ``routed_scaling_factor``); the two - overrides are the parameter dtypes and the logits GEMM. The correction - bias sits around magnitude ~10 while the sigmoid scores it corrects are - O(1e-2), so ranking turns on inter-expert gaps of 4e-5 - 6e-4; bf16 - resolution at that magnitude is ~1e-2, three orders of magnitude too - coarse -- it silently changes the top-8 while every aggregate check still - passes. The router weight is FP32 in the checkpoint, so the logits are an - FP32 ``F.linear`` (the shared bf16 router GEMM does not apply). + """DeepSeek noaux_tc routing with FP32 weights, logits and correction bias. + + Preserve small inter-expert score differences when adding the correction bias; + rounding it to BF16 can change expert selection. """ def __init__(self, config: PretrainedConfig, moe_backend: str = "CUTLASS") -> None: @@ -1663,24 +1573,11 @@ def forward( def glm5_next_hyper_connection( config: PretrainedConfig, dtype: torch.dtype = torch.bfloat16 ) -> mHC: - """Build the shared ``mHC`` for one hyper-connection site. - - This reuses TensorRT-LLM's existing manifold-constrained hyper-connection - rather than adding a model-local one. Two settings are model-specific and - were pinned by measurement against the source module, not by assumption: - - * ``post_mult_value=2.0`` -- the source computes ``2 * sigmoid(...)`` for the - block-output placement weights. Leaving the default 1.0 halves them and - shows up as ``max_abs`` 0.96 on ``post`` against a [0.26, 1.92] range. - * ``sinkhorn_iters`` is the config's own ``hc_sinkhorn_iters`` (20), not - ``iters - 1``. The source runs one initial column normalization plus - ``iters - 1`` row/column rounds, which is what this kernel's ``iters`` - counts; passing 19 leaves a measurable 1.9e-5 residual on ``comb`` versus - 1.3e-6 at 20. - - ``norm_eps`` is the decoder's ``rms_norm_eps`` because the source's - hyper-connection input norm is constructed with it, while ``eps`` and - ``sinkhorn_eps`` are the separate ``hc_eps``. + """Configure shared mHC with GLM's normalization and mixing semantics. + + Post weights are 2 * sigmoid(...). Pass the full hc_sinkhorn_iters count: + the kernel includes the initial column normalization. Input normalization uses + rms_norm_eps; mixing and Sinkhorn use hc_eps. """ from ..modules.mhc.hyper_connection import mHC @@ -1880,13 +1777,10 @@ def run_mlp(self, x: torch.Tensor, all_rank_num_tokens: list[int] | None) -> tor class Glm5NextModel(DecoderModel): - """Embedding, the 45 hyper-connected decoder layers, and the final readout. + """Carry [tokens, hc_mult, hidden] streams through the decoder layers. - A ``DecoderModel`` whose hidden state between layers is the four-stream - tensor ``[num_tokens, hc_mult, hidden]`` rather than ``[num_tokens, - hidden]``: the stream axis opens at the embedding and closes in - :meth:`collapse_streams` (unweighted mean plus the final norm), which is - this model's equivalent of the base class's trailing ``self.norm``. + Expand embeddings into streams, then collapse with an unweighted mean and + final RMSNorm. """ def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: @@ -1894,15 +1788,8 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: super().__init__(model_config) config = unwrap_glm5_next_text_config(model_config.pretrained_config) schedule = resolve_glm5_next_schedule(model_config.pretrained_config) - # Pipeline parallelism rides the base machinery wholesale: the - # inter-layer activation is the four-stream tensor [tokens, hc_mult, - # hidden], and both `forward_after_recv`/`forward_before_send` and - # `pp_recv/send` are shape-agnostic — the recv buffer on a non-first - # rank is exactly `expand_streams(embed_tokens.skip_forward(...))`, - # which is a real contiguous [tokens, hc_mult, hidden] tensor. - # `__pp_init__` prunes non-local layers; `load_weights` skips - # pruned owners (see `skipped_remote`); the hybrid cache manager - # slices its layer masks per rank from `mapping`. + # PP transports contiguous [tokens, hc_mult, hidden] streams. The base + # class prunes remote layers, which load_weights skips on this rank. self.config = config self.schedule = schedule self.hc_mult = int(config.hc_mult) @@ -2048,16 +1935,10 @@ def collapse_streams(self, hidden_streams: torch.Tensor) -> torch.Tensor: class Glm5NextMTPHead(nn.Module): - """The MTP layer's ``shared_head``: its final norm plus the draft logits. - - ``norm`` is applied by :class:`Glm5NextMTP` at the end of its forward (the - checkpoint's ``shared_head.norm``); ``forward`` here turns the resulting - hidden states into logits through the *target's* ``lm_head`` -- the - checkpoint publishes no separate MTP head weight -- the way the - speculative worker calls it: ``shared_head(hidden, lm_head, attn_metadata, - return_context_logits)``. The LM-head TP handling mirrors the DeepSeek-V3 - MTP head exactly, because the worker's greedy draft sampler recovers the - global argmax from vocab-sharded logits (``is_spec_decoding_head``). + """Normalize MTP hidden states and project draft logits through the target head. + + Glm5NextMTP applies norm before this forward. The speculative sampler consumes + vocab-sharded logits; the checkpoint has no separate MTP head weight. """ def __init__( diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py index d08ca110ff2f..bc8015b44ee1 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py @@ -164,10 +164,8 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig], layer_idx: int) ), head_dim=config.hidden_size // config.num_heads, ) - # Hard proof, not just a name check: `get_attention_backend` falls - # back to TRTLLM on unknown names and wraps known ones for sparse - # configs, so verify the constructed backend object is exactly the - # plain TRTLLM implementation. + # Backend resolution can substitute an implementation; require plain + # TRTLLM metadata and full-attention semantics after construction too. if type(self.attn) is not TrtllmAttention: raise ValueError( f"{type(self).__name__}: constructed attention backend is " @@ -195,11 +193,7 @@ def forward( qkv = self.qkv_proj(hidden_states) q, k, v = self.split_qkv(qkv, None, None) seq_len = q.shape[0] - # Per-head Q/K RMSNorm on the shared module, then table-driven RoPE: - # the two-axis (height, width) layout is carried by per-token cos/sin - # rows, so the FlashInfer cos/sin-cache RoPE op applies it with - # positions = row index (the same path Qwen2.5-VL's tower uses); the - # torch helper is the fallback. + # Normalize per head, then apply the two-axis RoPE tables by token row. q = self.q_norm(q.reshape(-1, self.head_dim)).reshape(seq_len, -1) k = self.k_norm(k.reshape(-1, self.head_dim)).reshape(seq_len, -1) cos, sin = position_embeddings @@ -310,8 +304,7 @@ def __init__(self, config: PretrainedConfig, dtype: torch.dtype) -> None: def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: weight = self.proj.weight # kernel == stride == the packed patch extent, so the Conv3d is exactly - # one GEMM over the flattened (C, T, P, P) patch rows (the cuDNN - # implicit-GEMM conv is ~10x slower than the cuBLAS GEMM here). + # one GEMM over the flattened (C, T, P, P) patch rows. hidden_states = hidden_states.reshape(-1, weight.shape[1:].numel()).to(dtype=weight.dtype) return nn.functional.linear(hidden_states, weight.view(self.embed_dim, -1), self.proj.bias) @@ -413,13 +406,8 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: self.downsample.bias = nn.Parameter(torch.empty(self.config.out_hidden_size, dtype=dtype)) self.merger = Glm5NextVisionPatchMerger(model_config) - # Source rotary: theta 10000 over half the head dim; the two position - # axes (h, w) each get head_dim//4 frequencies and the pair is - # duplicated over both head halves (neox pairing) — fp32 throughout. - # Deliberately NOT a module buffer: the encoder-wide dtype cast would - # downcast it to bf16 (the source keeps its non-persistent inv_freq - # table in fp32), and a 0.4% frequency error at grid positions ~50 - # is a ~1e-2 cos error that compounds across the 24 blocks. + # Two-axis Neox RoPE uses head_dim//4 frequencies per axis in FP32. + # Keep inv_freq outside module buffers so encoder dtype casts preserve it. freq_dim = self.head_dim // 2 # device='cpu' pins the table out of any meta-device construction # context (it is data, not a weight to materialize later). @@ -435,9 +423,7 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: ] = {} self._rope_cache_limit = 64 - # Pinned, not selected: the backend lock above already rejected every - # non-TRTLLM configuration, so the metadata class is the TRTLLM one by - # construction (no `get_attention_backend` name lookup on this path). + # The vision backend is fixed to plain TRTLLM. self.metadata_cls = TrtllmAttention.Metadata self.attn_metadata: Optional[AttentionMetadata] = None self._fixed_max_seq_len = model_config.max_num_tokens From 80a2ef9114ee33f1aad83bac9b28b14f39600eda Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Wed, 16 Sep 2026 06:44:31 -0700 Subject: [PATCH 09/35] [None][docs] Pin GLM-5.3-Flash deployment to Transformers 5.17.0 Replace the development revision with the released Transformers version in the GLM guide and supported-model entry. Shared requirements stay unchanged. Validated native config and image/video processing, full GSM8K with TP/MTP and ADP/MTP/FP8 KV, and video serving in the 5.17.0 deployment environment. Signed-off-by: Ruocheng Jia --- .../deployment-guide-for-glm-5.3-flash-on-trtllm.md | 6 +++--- docs/source/models/supported-models.md | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md index 342a31a6875e..0b5d8f6e7b35 100644 --- a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md @@ -35,10 +35,10 @@ See the [Model-Feature Support Matrix](../models/supported-models.md#model-featu * Drivers: CUDA Driver 575 or later * Docker with NVIDIA Container Toolkit installed * Minimum TensorRT LLM version: 1.3.0rc26 -* Install Transformers commit `49995e1a8c76de9158cf4f4e4995ac7454b8f141` in the GLM deployment environment for model configuration and image/video processing: +* Install `transformers==5.17.0` in the GLM deployment environment for model configuration and image/video processing: ```bash - pip install "git+https://github.com/huggingface/transformers.git@49995e1a8c76de9158cf4f4e4995ac7454b8f141" + pip install "transformers==5.17.0" ``` * Install `fla-core` and `einops` in the container: `pip install fla-core einops`. @@ -310,7 +310,7 @@ The checkpoint supports a total sequence length of 1,048,576 tokens. Set `--max_ ### Troubleshooting Tips * **CUDA OOM errors:** Reduce `--max_batch_size`, `--max_num_tokens`, or `kv_cache_config.free_gpu_memory_fraction`. Keep `cuda_graph_config.max_batch_size` consistent with the server batch limit. -* **Configuration or processor errors:** Check that the Transformers revision in [Prerequisites](#prerequisites) is installed in the serving environment. +* **Configuration or processor errors:** Check that the Transformers version in [Prerequisites](#prerequisites) is installed in the serving environment. * **Block reuse disabled:** Configure the periodic snapshot policy described in [`kv_cache_config`](#kv_cache_config). ## Benchmarking Performance diff --git a/docs/source/models/supported-models.md b/docs/source/models/supported-models.md index 7b32a6ce9dfb..d644a2007ee4 100644 --- a/docs/source/models/supported-models.md +++ b/docs/source/models/supported-models.md @@ -96,7 +96,7 @@ statuses for the same architecture in the two matrices. [^17]: Kimi K3 has no MTP or EAGLE-3 head, and its DSpark checkpoints are not compatible with plain `DFlash`. [^18]: NGram and standalone Suffix Automaton (SA) use model-free drafting on the PyTorch backend, so they are not listed in individual entries. This does not imply universal end-to-end support: compatibility depends on each model's multi-token verification and cache-management paths and may be untested or explicitly restricted. [^19]: KV cache reuse for hybrid recurrent-attention models requires an explicit recurrent-state snapshot policy, such as `kv_cache_config.mamba_state_config.periodic_snapshot_interval`; the model default disables reuse when no snapshot policy is configured. -[^21]: Supports text, image and video inputs and one-model MTP with 1 to 5 draft tokens. Requires the GLM-specific Transformers revision documented in the deployment guide, whose `Glm5NextProcessor` performs the image and video preprocessing. Supports FP8 KV cache and attention data parallelism with MTP. Beam search is not supported; disaggregated serving requires the Python NIXL transceiver; KV cache block reuse requires `kv_cache_config.mamba_state_config.periodic_snapshot_interval`. See the [deployment guide](../deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md). +[^21]: Supports text, image and video inputs and one-model MTP with 1 to 5 draft tokens. Requires `transformers==5.17.0` as documented in the deployment guide, whose `Glm5NextProcessor` performs the image and video preprocessing. Supports FP8 KV cache and attention data parallelism with MTP. Beam search is not supported; disaggregated serving requires the Python NIXL transceiver; KV cache block reuse requires `kv_cache_config.mamba_state_config.periodic_snapshot_interval`. See the [deployment guide](../deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md). # Encoder-Decoder Feature Support Matrix (PyTorch Backend) From 679572786ea38e06abb8ac67ed155380ae506392 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Wed, 16 Sep 2026 06:45:13 -0700 Subject: [PATCH 10/35] [None][refactor] Consolidate GLM sparse attention and KDA runtime setup Move packed-batch phase dispatch into sparse attention and share KDA hybrid cache construction with Kimi. Reuse sequential verification when the fused kernel is unavailable, retaining the optimized replay path by default. Honor the standard all-reduce option with an ONESHOT model default while preserving the measured large-message NCCL crossover. Simplify checkpoint auditing and config handling, avoid copying live KV lengths, and read vision profiling bounds from the processor configuration. Guard the unsupported MTP/attention-DP/tensor-parallel LM-head combination, update contract tests without adding CI stages, and regenerate the required LLM argument manifest. Validate shared-version CPU tests, 5.17.0 B200 tests, full-model accuracy, video serving, and real V2 sequential verify/replay. Signed-off-by: Ruocheng Jia --- .../checkpoints/hf/glm5_next_weight_mapper.py | 77 +---- .../_torch/models/modeling_glm5_next.py | 300 +++++++----------- .../models/modeling_glm5_next_vision.py | 19 +- tensorrt_llm/_torch/pyexecutor/_util.py | 144 ++------- .../_torch/pyexecutor/config_utils.py | 12 +- .../modeling/test_glm5_next_contracts.py | 141 ++++++-- 6 files changed, 282 insertions(+), 411 deletions(-) diff --git a/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py b/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py index 47d2804c14e8..4f575ccf56c2 100644 --- a/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py +++ b/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py @@ -20,8 +20,7 @@ from __future__ import annotations import re -from collections import Counter -from collections.abc import Iterable, Sequence +from collections.abc import Iterable from dataclasses import dataclass, field from typing import Any @@ -48,48 +47,15 @@ _HC_RE = re.compile(r"^(model\.layers\.\d+\.)hc_(attn|ffn)_(fn|base|scale)$") -class Disposition: - """How a checkpoint tensor reaches (or does not reach) the runtime.""" - - #: Placed on a destination parameter unchanged. - LOADED = "loaded" - #: Routed through expert packing or scale-layout conversion. - TRANSFORMED = "transformed" - #: Deliberately not loaded, under an exact allowlisted namespace. - IGNORED = "ignored" - - @dataclass class Glm5NextWeightAudit: - """Exhaustive per-key accounting for a GLM-5.3-Flash checkpoint.""" + """Checkpoint destinations, explicitly ignored keys and unresolved keys.""" - #: destination module/parameter name -> source key + # Runtime destination -> checkpoint source key. destinations: dict[str, str] = field(default_factory=dict) - #: source key -> disposition - disposition: dict[str, str] = field(default_factory=dict) - #: source key -> why it was transformed / ignored - reason: dict[str, str] = field(default_factory=dict) - #: keys that could not be placed at all -- always a hard error + ignored: set[str] = field(default_factory=set) unresolved: list[str] = field(default_factory=list) - def counts(self) -> dict[str, int]: - return dict(Counter(self.disposition.values())) - - def keys_with(self, disposition: str) -> list[str]: - return sorted(k for k, d in self.disposition.items() if d == disposition) - - def reasons(self) -> dict[str, int]: - return dict(Counter(self.reason.values())) - - -def _ignored_reason(key: str, mtp_prefixes: Sequence[str]) -> str | None: - if key.startswith(_VISION_PREFIX): - return "vision tower weights are loaded by Glm5NextVLM, not the text decoder" - for prefix in mtp_prefixes: - if key.startswith(prefix): - return "MTP / next-n prediction layer is not enabled (no speculative_config)" - return None - def remap_glm5_next_key(key: str) -> str | None: """Map one text-decoder checkpoint key to its runtime destination. @@ -141,31 +107,14 @@ def audit_glm5_next_checkpoint( audit = Glm5NextWeightAudit() for key in keys: - ignored = _ignored_reason(key, mtp_prefixes) - if ignored is not None: - audit.disposition[key] = Disposition.IGNORED - audit.reason[key] = ignored + if key.startswith((_VISION_PREFIX, *mtp_prefixes)): + audit.ignored.add(key) continue - dest = remap_glm5_next_key(key) if dest is None: audit.unresolved.append(key) - continue - - if dest.endswith(".weight_scale_inv"): - audit.disposition[key] = Disposition.TRANSFORMED - audit.reason[key] = "block-FP8 128x128 weight scale" + else: audit.destinations[dest] = key - continue - - if ".mlp.experts." in dest: - audit.disposition[key] = Disposition.TRANSFORMED - audit.reason[key] = "routed expert stacked into the fused MoE layout" - audit.destinations[dest] = key - continue - - audit.disposition[key] = Disposition.LOADED - audit.destinations[dest] = key return audit @@ -191,7 +140,7 @@ def glm5_next_is_quantized(model_config: ModelConfig[PretrainedConfig]) -> bool: return True -def _destination_owner(dest: str, num_layers: int) -> Any: +def glm5_next_weight_owner(dest: str, num_layers: int) -> Any: """Which materialization unit owns ``dest``: a layer index or a named part.""" match = re.match(r"^model\.layers\.(\d+)\.", dest) if match is not None: @@ -221,13 +170,3 @@ def audit(self, keys: Iterable[str]) -> Glm5NextWeightAudit: return audit_glm5_next_checkpoint( keys, self.config.pretrained_config, num_mtp_layers=num_mtp_layers ) - - @staticmethod - def destination(key: str) -> str | None: - """Runtime parameter name for a checkpoint key (``None``: not loaded).""" - return remap_glm5_next_key(key) - - @staticmethod - def owner(dest: str, num_layers: int) -> Any: - """Materialization owner of a destination: a layer index or a named part.""" - return _destination_owner(dest, num_layers) diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next.py b/tensorrt_llm/_torch/models/modeling_glm5_next.py index 2916aa372e18..7075565ded57 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next.py @@ -21,7 +21,7 @@ from __future__ import annotations -import os +import copy import re from collections.abc import Sequence from dataclasses import dataclass, field @@ -45,6 +45,7 @@ GlmKpoolSparseParams, ) from ..attention.backends.utils import create_attention +from ..distributed import AllReduceStrategy from ..model_config import ModelConfig from ..modules.decoder_layer import DecoderLayer from ..modules.embedding import Embedding @@ -55,16 +56,15 @@ from ..modules.rms_norm import RMSNorm from ..pyexecutor.config_utils import unwrap_glm5_next_text_config from .checkpoints.hf.glm5_next_weight_mapper import ( - Disposition, Glm5NextHfWeightMapper, glm5_next_is_quantized, + glm5_next_weight_owner, ) from .modeling_deepseekv3 import DeepseekV3Gate from .modeling_speculative import SpecDecOneEngineForCausalLM from .modeling_utils import DecoderModel, register_auto_model if TYPE_CHECKING: - from ..distributed import AllReduceStrategy from ..modules.mamba.mamba2_metadata import Mamba2Metadata from ..modules.mhc.hyper_connection import mHC from .checkpoints.base_weight_mapper import BaseWeightMapper @@ -150,24 +150,6 @@ def resolve_glm5_next_schedule(config: PretrainedConfig) -> Glm5NextSchedule: return schedule -def glm5_next_allreduce_strategy() -> AllReduceStrategy: - """Select the small-message TP all-reduce strategy (default: ONESHOT). - - TLLM_GLM5_ALLREDUCE overrides the strategy for tuning. Large messages use NCCL - regardless of this setting; a fixed strategy avoids AUTO's runtime autotuning. - """ - from ..distributed import AllReduceStrategy - - name = os.environ.get("TLLM_GLM5_ALLREDUCE", "ONESHOT").strip().upper() - try: - return AllReduceStrategy[name] - except KeyError as exc: - raise ValueError( - f"TLLM_GLM5_ALLREDUCE={name!r} is not an AllReduceStrategy name: " - f"{[m.name for m in AllReduceStrategy]}" - ) from exc - - def glm5_next_tp_reduces(mapping: Mapping | None) -> bool: """Whether a TP branch output is a partial that needs one all-reduce. @@ -201,21 +183,27 @@ def glm5_next_attention_mapping(mapping: Mapping | None) -> Mapping | None: class Glm5NextAllReduce(nn.Module): """Use the configured TP collective for small messages and NCCL for large ones. - Dispatch depends only on tensor shape, so CUDA graph capture fixes the choice. + The 512-token ONESHOT/NCCL crossover is tuned for TP4 BF16 messages; + it is independent of the C++ workspace-capacity fallback. Other explicitly + selected strategies apply at every size. Dispatch is fixed at graph capture. """ # TP4/BF16 tuning uses fused collectives up to 512 tokens, then NCCL. SMALL_MAX_TOKENS = 512 - def __init__(self, mapping: Mapping, dtype: torch.dtype = torch.bfloat16) -> None: + def __init__( + self, + mapping: Mapping, + dtype: torch.dtype = torch.bfloat16, + strategy: AllReduceStrategy = AllReduceStrategy.ONESHOT, + ) -> None: super().__init__() - from ..distributed import AllReduce, AllReduceStrategy + from ..distributed import AllReduce - strategy = glm5_next_allreduce_strategy() self.small = AllReduce(mapping=mapping, strategy=strategy, dtype=dtype) self.large = ( self.small - if strategy == AllReduceStrategy.NCCL + if strategy != AllReduceStrategy.ONESHOT else AllReduce(mapping=mapping, strategy=AllReduceStrategy.NCCL, dtype=dtype) ) @@ -333,37 +321,6 @@ def context_rows(self, kpool: int, device: torch.device) -> Glm5NextContextRows: self.ctx_rows_cache[kpool] = rows return rows - @property - def gen_phase(self) -> str: - """Select single-token decode or multi-token speculative verification.""" - return "decode" if self.gen_tokens_per_request == 1 else "verify" - - def mixed_kwargs(self) -> dict[str, Any]: - """Split sparse attention by phase while keeping the rest of the batch together.""" - return { - "num_ctx_tokens": self.num_ctx_tokens, - "prefill": self.sparse_kwargs("prefill"), - "generation": self.sparse_kwargs(self.gen_phase), - "generation_phase": self.gen_phase, - } - - def sparse_kwargs(self, phase: str) -> dict[str, Any]: - # Schedule only: the backend (keyed by its own layer_idx) derives the - # slot-indexed latent/indexer views, block tables, and lengths from - # the prepared metadata itself. - kwargs: dict[str, Any] = {"metadata": self.metadata} - if phase == "prefill": - kwargs.update( - cached_lens=self.cached_lens[: self.num_contexts], - cu_seqlens=self.ctx_cu_seqlens, - ctx_rows_fn=self.context_rows, - ) - else: - kwargs.update(kv_lens=self.kv_lens[self.num_contexts :]) - if phase == "verify": - kwargs.update(tokens_per_request=self.gen_tokens_per_request) - return kwargs - def _glm5_gen_tokens_per_request(attn_metadata: AttentionMetadata, num_generations: int) -> int: """Tokens per generation request, from the metadata's host token counts. @@ -414,7 +371,7 @@ def build_glm5_next_runtime_context(attn_metadata: AttentionMetadata) -> Glm5Nex num_generations=num_generations, ctx_cu_seqlens=mamba_metadata.glm_ctx_cu_seqlens, cached_lens=mamba_metadata.glm_cached_lens_host, - kv_lens=live_lengths[:batch].to(torch.long), + kv_lens=live_lengths[:batch], metadata=attn_metadata, gen_tokens_per_request=_glm5_gen_tokens_per_request(attn_metadata, num_generations), ) @@ -428,12 +385,26 @@ class Glm5NextForCausalLM(SpecDecOneEngineForCausalLM): routes text and MTP tensors and excludes the vision namespace. """ + @classmethod + def get_model_defaults(cls, llm_args) -> dict: + # Avoid AUTO autotuning on TP decode; honor explicit user overrides. + return {"allreduce_strategy": "ONESHOT"} + @property def mamba_metadata_cls(self) -> type[Mamba2Metadata]: """Metadata with paged tables refreshed before CUDA graph replay.""" return Glm5NextMamba2Metadata def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: + if ( + model_config.mapping.enable_attention_dp + and model_config.mapping.enable_lm_head_tp_in_adp + and model_config.spec_config is not None + and model_config.spec_config.spec_dec_mode.is_mtp_one_model() + ): + raise NotImplementedError( + "glm5_next MTP with attention DP and a tensor-parallel LM head is not supported" + ) text_config = unwrap_glm5_next_text_config(model_config.pretrained_config) # tie_word_embeddings is false on this checkpoint, so lm_head is a real # weight rather than a view of the embedding; it is asserted, not assumed. @@ -548,8 +519,6 @@ def load_weights( self, weights: Any, weight_mapper: BaseWeightMapper | None = None, - *, - device_map: dict[Any, Any] | None = None, ) -> None: """Load checkpoint tensors one owner at a time to bound peak memory. @@ -560,9 +529,6 @@ def load_weights( Args: weights: Mapping of checkpoint keys to tensors or lazy safetensors slices. weight_mapper: Initialized Glm5NextHfWeightMapper, or None to create one. - device_map: Optional owner-to-device map. Owners are layer indices or - "embed", "norm", and "head". Modules are materialized from meta - directly onto their target devices. """ if weight_mapper is None: weight_mapper = Glm5NextHfWeightMapper() @@ -602,11 +568,8 @@ def load_weights( } by_owner: dict[Any, list[tuple[str, str]]] = {} - for key, disposition in audit.disposition.items(): - if disposition == Disposition.IGNORED: - continue - dest = weight_mapper.destination(key) - owner = weight_mapper.owner(dest, num_layers) if dest else None + for dest, key in audit.destinations.items(): + owner = glm5_next_weight_owner(dest, num_layers) if owner is None: raise ValueError(f"glm5_next has no destination owner for {key!r} -> {dest!r}") if owner in remote: @@ -615,11 +578,10 @@ def load_weights( if audit.unresolved: raise ValueError(f"glm5_next cannot place {sorted(audit.unresolved)[:5]}") - default_device = torch.device("cuda", torch.cuda.current_device()) + device = torch.device("cuda", torch.cuda.current_device()) for owner, (module, prefix) in targets.items(): if owner in remote: continue - device = torch.device((device_map or {}).get(owner, default_device)) module.to_empty(device=device) self._fill_module(module, owner, prefix, by_owner.get(owner, []), weights, device) # The shared KDA mixer's post-load kernel constants (built from the @@ -829,19 +791,20 @@ def __init__( layer_idx: int, dtype: torch.dtype = torch.bfloat16, mapping: Mapping | None = None, + allreduce_strategy: AllReduceStrategy = AllReduceStrategy.ONESHOT, ) -> None: del dtype # the shared mixer is bf16 (fp32 gate parameters and pools) # The checkpoint's ``linear_attn_config`` does not name the gate rank: # GLM-5.3-Flash's output gate is always the low-rank ``g_a``/``g_b`` # pair, which the shared mixer reads from the same key Kimi K3 uses. + config = copy.copy(config) + config.linear_attn_config = dict(config.linear_attn_config) config.linear_attn_config.setdefault("use_full_rank_gate", False) - linear = dict(config.linear_attn_config) + linear = config.linear_attn_config # The mixer's own row-parallel o_proj AllReduce (fixed strategy; the # size-aware Glm5NextAllReduce used elsewhere is a perf option, not a # correctness requirement, and AUTO's autotuner raced at TP4 decode). - super().__init__( - config, layer_idx, mapping=mapping, allreduce_strategy=glm5_next_allreduce_strategy() - ) + super().__init__(config, layer_idx, mapping=mapping, allreduce_strategy=allreduce_strategy) # Preserve checkpoint FP32 gate parameters. MetaInitMode rejects detach # on meta tensors, so create empty FP32 parameters on that branch. for name in ("A_log", "dt_bias"): @@ -909,7 +872,11 @@ def _glm5_linear_kwargs( "skip_create_weights_in_init": ( model_config.skip_create_weights_in_init if model_config is not None else False ), - "allreduce_strategy": glm5_next_allreduce_strategy(), + "allreduce_strategy": ( + model_config.allreduce_strategy + if model_config is not None + else AllReduceStrategy.ONESHOT + ), "disable_deep_gemm": True, } @@ -1091,7 +1058,16 @@ def __init__( reduce_output=False, **lin_kwargs, ) - self.tp_all_reduce = Glm5NextAllReduce(mapping) if glm5_next_tp_reduces(mapping) else None + self.tp_all_reduce = ( + Glm5NextAllReduce( + mapping, + strategy=model_config.allreduce_strategy + if model_config is not None + else AllReduceStrategy.ONESHOT, + ) + if glm5_next_tp_reduces(mapping) + else None + ) self.indexer = Glm5NextIndexer( config, layer_idx, mapping=mapping, model_config=model_config ) @@ -1335,22 +1311,52 @@ def forward_decode( reduce=reduce, ) - def forward_mixed( + def forward( self, hidden_states: torch.Tensor, + attn_metadata: AttentionMetadata, *, - num_ctx_tokens: int, - prefill: dict[str, Any], - generation: dict[str, Any], - generation_phase: str = "decode", + runtime_ctx: Glm5NextRuntimeContext | None = None, ) -> torch.Tensor: - """Split attention by phase and reduce the concatenated output once.""" - if generation_phase not in ("decode", "verify"): - raise ValueError(f"Unsupported generation phase: {generation_phase}") - ctx = self.forward_prefill(hidden_states[:num_ctx_tokens], reduce=False, **prefill) - generation_forward = getattr(self, f"forward_{generation_phase}") - gen = generation_forward(hidden_states[num_ctx_tokens:], reduce=False, **generation) - out = torch.cat([ctx, gen], dim=0) + """Process a packed batch, splitting phases only inside attention. + + Reuse the model's per-forward schedules and row cache across layers. + Reduce the complete output once, including context/verify mixtures. + """ + ctx = ( + runtime_ctx + if runtime_ctx is not None + else build_glm5_next_runtime_context(attn_metadata) + ) + parts = [] + if ctx.num_contexts: + parts.append( + self.forward_prefill( + hidden_states[: ctx.num_ctx_tokens], + cu_seqlens=ctx.ctx_cu_seqlens, + cached_lens=ctx.cached_lens[: ctx.num_contexts], + metadata=ctx.metadata, + ctx_rows_fn=ctx.context_rows, + reduce=False, + ) + ) + if ctx.num_generations: + generation = hidden_states[ctx.num_ctx_tokens :] + lengths = ctx.kv_lens[ctx.num_contexts :] + if ctx.gen_tokens_per_request == 1: + out = self.forward_decode( + generation, kv_lens=lengths, metadata=ctx.metadata, reduce=False + ) + else: + out = self.forward_verify( + generation, + kv_lens=lengths, + metadata=ctx.metadata, + tokens_per_request=ctx.gen_tokens_per_request, + reduce=False, + ) + parts.append(out) + out = parts[0] if len(parts) == 1 else torch.cat(parts, dim=0) return out if self.tp_all_reduce is None else self.tp_all_reduce(out) def forward_verify( @@ -1519,7 +1525,9 @@ def __init__( # attention DP the fused layer combines across ranks itself and the # shared expert is replicated, so there is nothing to reduce here. self.moe_all_reduce = ( - Glm5NextAllReduce(self.mapping) if glm5_next_tp_reduces(self.mapping) else None + Glm5NextAllReduce(self.mapping, strategy=model_config.allreduce_strategy) + if glm5_next_tp_reduces(self.mapping) + else None ) # Shared expert: TP-sharded over ``model_config.mapping`` with # ``reduce_output=False``, so its output is a rank partial summed with @@ -1620,8 +1628,7 @@ class Glm5NextDecoderLayer(DecoderLayer): Each connection collapses [tokens, hc_mult, hidden] streams for the sublayer and mixes its output back into the streams. Attention and MLP types come from - the explicit per-layer schedules. forward derives runtime arguments; - forward_direct applies the same math with explicit attention arguments. + the explicit per-layer schedules. """ def __init__( @@ -1641,7 +1648,11 @@ def __init__( mapping = model_config.mapping if self.attention_type == LINEAR_ATTENTION: self.self_attn: nn.Module = Glm5NextLinearAttention( - config, layer_idx, dtype=dtype, mapping=mapping + config, + layer_idx, + dtype=dtype, + mapping=mapping, + allreduce_strategy=model_config.allreduce_strategy, ) else: self.self_attn = Glm5NextSparseAttention( @@ -1672,7 +1683,7 @@ def __init__( ) ) self.mlp_all_reduce = ( - Glm5NextAllReduce(mapping) + Glm5NextAllReduce(mapping, strategy=model_config.allreduce_strategy) if self.mlp_type != SPARSE_MLP and glm5_next_tp_reduces(mapping) else None ) @@ -1691,79 +1702,25 @@ def forward( runtime_ctx: Glm5NextRuntimeContext | None = None, **kwargs: Any, ) -> torch.Tensor: - """Runtime entry: derive this layer's cache arguments from metadata. - - The executor packs context requests first, then generation requests; - attention handles the two phases separately, while the remaining - operations run once over the packed batch to preserve DP collectives. - ``position_ids`` is unused: the text path is fully NoPE and the - indexer derives its positions from the cached lengths. - """ + """Run attention and one full-batch FFN between the mHC connections.""" if runtime_ctx is None: runtime_ctx = build_glm5_next_runtime_context(attn_metadata) - all_rank_num_tokens = getattr(runtime_ctx.metadata, "all_rank_num_tokens", None) - if self.attention_type == LINEAR_ATTENTION: - # The shared KDA mixer splits context / generation rows (prefill, - # decode, verify) itself from the prepared metadata. - return self.forward_direct( - hidden_states, - all_rank_num_tokens=all_rank_num_tokens, - attn_metadata=runtime_ctx.metadata, - ) - derive = runtime_ctx.sparse_kwargs - if runtime_ctx.num_contexts > 0 and runtime_ctx.num_generations > 0: - # Only attention is phase-specific. MoE must see the complete - # packed batch exactly once: DP ranks can have different phase - # mixes, but must issue the same collective sequence and counts. - return self.forward_direct( - hidden_states, - phase="mixed", - all_rank_num_tokens=all_rank_num_tokens, - **runtime_ctx.mixed_kwargs(), - ) - if runtime_ctx.num_contexts > 0: - return self.forward_direct( - hidden_states, - phase="prefill", - all_rank_num_tokens=all_rank_num_tokens, - **derive("prefill"), - ) - # "decode" (one token per request) or "verify" (golden + drafts per - # request while a speculative-decoding target scores them). - phase = runtime_ctx.gen_phase - return self.forward_direct( - hidden_states, - phase=phase, - all_rank_num_tokens=all_rank_num_tokens, - **derive(phase), - ) - - def forward_direct( - self, - hidden_streams: torch.Tensor, - phase: str = "prefill", - all_rank_num_tokens: list[int] | None = None, - **attn_kwargs: Any, - ) -> torch.Tensor: - """``hidden_streams`` is ``[num_tokens, hc_mult, hidden]``. - - A KDA layer takes ``attn_metadata`` and runs the shared mixer over all - rows; a sparse layer's ``attn_kwargs`` are forwarded verbatim to its - ``forward_``. ``all_rank_num_tokens`` (attention DP) reaches the - fused MoE. - """ - residual = hidden_streams - post, comb, collapsed = self.hc_attn.pre_mapping(hidden_streams) + metadata = runtime_ctx.metadata + residual = hidden_states + post, comb, collapsed = self.hc_attn.pre_mapping(hidden_states) normed = self.input_layernorm(collapsed) if self.attention_type == LINEAR_ATTENTION: - attn_out = self.self_attn(normed, attn_kwargs["attn_metadata"]) + attn_out = self.self_attn(normed, metadata) else: - attn_out = getattr(self.self_attn, f"forward_{phase}")(normed, **attn_kwargs) - hidden_streams = self.hc_attn.post_mapping(attn_out, residual, post, comb) + attn_out = self.self_attn(normed, metadata, runtime_ctx=runtime_ctx) + hidden_states = self.hc_attn.post_mapping(attn_out, residual, post, comb) - residual = hidden_streams - post, comb, collapsed = self.hc_ffn.pre_mapping(hidden_streams) - mlp_out = self.run_mlp(self.post_attention_layernorm(collapsed), all_rank_num_tokens) + residual = hidden_states + post, comb, collapsed = self.hc_ffn.pre_mapping(hidden_states) + # ADP ranks may have different phase mixes; every rank calls MoE once. + mlp_out = self.run_mlp( + self.post_attention_layernorm(collapsed), getattr(metadata, "all_rank_num_tokens", None) + ) return self.hc_ffn.post_mapping(mlp_out, residual, post, comb) def run_mlp(self, x: torch.Tensor, all_rank_num_tokens: list[int] | None) -> torch.Tensor: @@ -1784,7 +1741,6 @@ class Glm5NextModel(DecoderModel): """ def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: - _normalize_glm5_next_top_config(model_config.pretrained_config) super().__init__(model_config) config = unwrap_glm5_next_text_config(model_config.pretrained_config) schedule = resolve_glm5_next_schedule(model_config.pretrained_config) @@ -1877,13 +1833,10 @@ def _forward_single_phase_fused_hc( previous site, ``pre_mapping`` of the next, and the next sublayer's RMSNorm; ``mHC.fused_hc`` (the in-tree DeepSeek-V4 boundary op) runs the three as one kernel, so a layer costs 2 fused boundaries instead - of 4 mappings + 2 norms. Same math as :meth:`Glm5NextDecoderLayer. - forward_direct` chained over the stack; the first pre-mapping and the - last post-mapping stay unfused. Only generation-only batches use it - (see :meth:`forward`); the phase is still derived here so the loop - stays valid for a pure-context batch. + of 4 mappings + 2 norms. The math matches the per-layer forward; the + first pre-mapping and last post-mapping stay unfused. Only generation-only batches use it + (see :meth:`forward`). """ - phase = "prefill" if runtime_ctx.num_generations == 0 else runtime_ctx.gen_phase num_layers = self.schedule.num_layers first = self.layers[0] residual = streams @@ -1894,9 +1847,7 @@ def _forward_single_phase_fused_hc( if layer.attention_type == LINEAR_ATTENTION: attn_out = layer.self_attn(x, runtime_ctx.metadata) else: - attn_out = getattr(layer.self_attn, f"forward_{phase}")( - x, **runtime_ctx.sparse_kwargs(phase) - ) + attn_out = layer.self_attn(x, runtime_ctx.metadata, runtime_ctx=runtime_ctx) residual, post, comb, x = layer.hc_ffn.fused_hc( attn_out, residual, @@ -2004,7 +1955,6 @@ def __init__( raise NotImplementedError( "glm5_next MTP runs one-model speculative decoding only (MTP / MTP_EAGLE_ONE_MODEL)" ) - _normalize_glm5_next_top_config(model_config.pretrained_config) config = unwrap_glm5_next_text_config(model_config.pretrained_config) schedule = resolve_glm5_next_schedule(model_config.pretrained_config) num_nextn = int(getattr(config, "num_nextn_predict_layers", 0) or 0) @@ -2071,13 +2021,7 @@ def forward( runtime_ctx = build_glm5_next_runtime_context(attn_metadata) residual = hidden_states attn_in = self.input_layernorm(hidden_states) - if runtime_ctx.num_contexts > 0 and runtime_ctx.num_generations > 0: - attn_out = self.self_attn.forward_mixed(attn_in, **runtime_ctx.mixed_kwargs()) - else: - phase = "prefill" if runtime_ctx.num_contexts > 0 else runtime_ctx.gen_phase - attn_out = getattr(self.self_attn, f"forward_{phase}")( - attn_in, **runtime_ctx.sparse_kwargs(phase) - ) + attn_out = self.self_attn(attn_in, attn_metadata, runtime_ctx=runtime_ctx) hidden_states = residual + attn_out residual = hidden_states diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py index bc8015b44ee1..df0928009858 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py @@ -9,7 +9,7 @@ Vision attention uses the TRTLLM backend with per-item full-attention segments and no KV cache, reusing Qwen-VL metadata preparation. The HF Glm5NextProcessor handles preprocessing and placeholder expansion; the required Transformers -revision is documented in the deployment guide. +version is documented in the deployment guide. """ import copy @@ -775,7 +775,7 @@ def _glm5_next_build_batched_input(multimodal_params: List[MultimodalParams]) -> class Glm5NextInputProcessor(BaseMultimodalInputProcessor, BaseMultimodalDummyInputsBuilder): """Adapt HF image/video preprocessing and placeholder expansion to engine inputs. - Requires the GLM-specific Transformers revision in the deployment guide. + Requires the Transformers version specified in the deployment guide. """ def __init__( @@ -888,14 +888,9 @@ def _max_grid_side(self, max_patches: int) -> int: return max(side, self._merge_size) def _processor_max_patches(self) -> int: - """The HF image processor's own patch cap (its ``max_pixels`` budget), - measured by asking it about an oversized square image.""" - big = 64 * 1024 - return int( - self._processor._get_num_multimodal_tokens(image_sizes=[(big, big)])[ - "num_image_patches" - ][0] - ) + """HF's image-token budget expressed in pre-merge patch rows.""" + processor = self._processor.image_processor + return int(processor.max_image_tokens) * int(processor.merge_size) ** 2 def get_mm_max_tokens_per_item( self, max_num_encoder_tokens: Optional[int] = None @@ -1065,6 +1060,10 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig], *args, **kwargs) self.post_config() # -- engine contracts ----------------------------------------------------- + @classmethod + def get_model_defaults(cls, llm_args) -> dict: + return Glm5NextForCausalLM.get_model_defaults(llm_args) + @classmethod def get_preferred_kv_cache_manager_version(cls, pretrained_config=None) -> str: return Glm5NextForCausalLM.get_preferred_kv_cache_manager_version(pretrained_config) diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index e5a47ac9a497..c23fd91c5688 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -3002,107 +3002,20 @@ def _create_kv_cache_manager( if issubclass(kv_cache_manager_cls, MambaHybridCacheManagerV2): manager_extra_kwargs["is_disagg"] = is_disagg - if config_utils.is_glm5_next(config): - # GLM-5.3-Flash hybrid: KDA recurrent/conv states on the mamba side, - # sparse-MLA latent cache (num_kv_heads=1, head_dim = kv_lora_rank + - # qk_rope_head_dim, SELFKONLY) plus one Role.INDEX_KEY indexer buffer - # per sparse layer on the paged side, all owned by one - # Glm5NextCacheManager. Must come before the is_mla(...) route: the - # glm5_next text config carries MLA fields, but only 11 of its 45 - # layers are sparse MLA. - if max_beam_width > 1: - raise ValueError("glm5_next + beam search is not supported yet.") - if not estimating_kv_cache and kv_connector_manager is not None: - raise NotImplementedError( - "Connector manager is not supported for glm5_next.") - text_config = config_utils.unwrap_glm5_next_text_config(config) - mamba_params = extract_mamba_kv_cache_params( - config, - spec_config=spec_config, - quant_config=quant_config, - ) - mamba_layer_mask, full_attention_layer_mask = ( - _get_mamba_cache_layer_masks( - mamba_params, - mapping, - spec_config, - is_draft, - )) - num_mamba_layers = (0 if is_draft and mamba_params.num_draft_layers > 0 - else mamba_params.num_mamba_layers) - # The indexer state rides the same layer ids as the sparse latent - # pages; both are slot-addressed through the manager's slot-major - # views (see Glm5NextCacheManager.get_batch_slot_tables). - sparse_layer_ids = [ - i for i, is_sparse in enumerate(full_attention_layer_mask) - if is_sparse - ] - # KDA fused multi-token verify (trtllm::kda_mtp_decode, shared with - # Kimi K3): the per-slot replay caches replace the per-step - # intermediate verification buffers. glm5_next has no other verify - # path, so the kernel must be available when speculating. - glm_extra_kwargs = {} - if spec_config is not None: - from ..modules.kimi_kda._kda_kernels import \ - is_kda_mtp_verify_available - if not is_kda_mtp_verify_available(): - raise RuntimeError( - "glm5_next speculative decoding requires the fused KDA " - "verify kernel (trtllm::kda_mtp_decode, CUTLASS DSL), " - "which is unavailable on this device/build.") - glm_extra_kwargs["kda_replay_num_spec"] = ( - spec_config.tokens_per_gen_step - 1) - kv_cache_manager = kv_cache_manager_cls( - # mamba (KDA) cache parameters - mamba_params.state_size, - mamba_params.conv_kernel, - mamba_params.num_heads, - mamba_params.n_groups, - mamba_params.head_dim, - num_mamba_layers, - mamba_layer_mask, - mamba_params.dtype, - mamba_params.mamba_ssm_cache_dtype, - # kv cache parameters (sparse-MLA latent + indexer state) - kv_cache_config, - tensorrt_llm.bindings.internal.batch_manager.CacheType.SELFKONLY, - num_layers=sum(full_attention_layer_mask), - layer_mask=full_attention_layer_mask, - num_kv_heads=1, - head_dim=int(text_config.kv_lora_rank) + - int(getattr(text_config, "qk_rope_head_dim", 0) or 0), - tokens_per_block=tokens_per_block, - max_seq_len=max_seq_len, - max_num_tokens=max_num_tokens, - is_draft=is_draft, - max_batch_size=max_batch_size, - mapping=mapping, - dtype=kv_cache_dtype, - spec_config=spec_config, - is_estimating_kv_cache=estimating_kv_cache, - execution_stream=execution_stream, - sparse_layer_ids=sparse_layer_ids, - # One indexer cache row is [k | gate | pool key] (see - # Glm5NextIndexer.cache_state_dim). - index_state_dim=3 * int(text_config.index_head_dim), - # KDA's conv state is a [Q | K | V] concatenation whose three - # sections have identical width, i.e. the qwen3_next layout. - **_mamba_conv_layout_kwargs(kv_cache_manager_cls, "qwen3_next"), - **glm_extra_kwargs, - **manager_extra_kwargs, - ) - elif is_kimi_linear(config): - # Kimi K3 hybrid: KDA (Kimi Delta Attention) recurrent/conv states on - # the mamba side of the hybrid manager, absorbed-MQA MLA latent cache - # (num_kv_heads=1, head_dim = kv_lora_rank + qk_rope_head_dim, - # SELFKONLY) on the paged-KV side. Must come before the is_mla(...) - # route: the kimi_linear config carries MLA fields, but only 24 of - # its 93 layers are MLA. + if config_utils.is_glm5_next(config) or is_kimi_linear(config): + is_glm = config_utils.is_glm5_next(config) + text_config = config_utils.unwrap_glm5_next_text_config( + config) if is_glm else config + # Both families combine KDA state with latent MLA pages. GLM adds + # paged indexer state; both use sequential verify when replay is unavailable. if max_beam_width > 1: raise ValueError( + "glm5_next + beam search is not supported yet." if is_glm else "MambaHybridCacheManager + beam search is not supported yet.") if not estimating_kv_cache and kv_connector_manager is not None: raise NotImplementedError( + "Connector manager is not supported for glm5_next." + if is_glm else "Connector manager is not supported for MambaHybridCacheManager." ) mamba_params = extract_mamba_kv_cache_params( @@ -3119,31 +3032,29 @@ def _create_kv_cache_manager( )) num_mamba_layers = (0 if is_draft and mamba_params.num_draft_layers > 0 else mamba_params.num_mamba_layers) - # Kimi K3 KDA state sharding follows the attention-family TP - # semantics (Qwen3-Next pattern): replicated under attention-DP, - # head-sharded across tp_size otherwise. That is exactly the cache - # manager's own internal gate (`tp_size = 1 if enable_attention_dp - # else tp_size`, then num_heads / n_groups / conv_dim divide by - # it), so the params pass through unscaled. - # KDA fused multi-token verify (trtllm::kda_mtp_decode): when the - # kernel can run here, allocate the per-slot replay caches instead - # of the legacy per-step intermediate verification buffers. The - # kernel replays accepted drafts from these caches and commits - # states in place, replacing the intermediate-buffer + promotion - # flow for KDA layers. - kimi_extra_kwargs = {} + # Pass full KDA head counts: the manager applies attention-TP sharding. + kda_extra_kwargs = {} kda_replay_manager_types = (MixedMambaHybridCacheManager, MambaHybridCacheManagerV2) - if (spec_config is not None - and issubclass(kv_cache_manager_cls, kda_replay_manager_types)): + if (spec_config is not None and + (is_glm + or issubclass(kv_cache_manager_cls, kda_replay_manager_types))): from ..modules.kimi_kda._kda_kernels import \ is_kda_mtp_verify_available if is_kda_mtp_verify_available(): - kimi_extra_kwargs["kda_replay_num_spec"] = ( + kda_extra_kwargs["kda_replay_num_spec"] = ( spec_config.tokens_per_gen_step - 1) + if is_glm: + kda_extra_kwargs.update( + sparse_layer_ids=[ + i for i, sparse in enumerate(full_attention_layer_mask) + if sparse + ], + index_state_dim=3 * int(text_config.index_head_dim), + ) # KDA's conv state is a [Q | K | V] concatenation whose three sections # have identical width, i.e. the qwen3_next section layout. - kimi_extra_kwargs.update( + kda_extra_kwargs.update( _mamba_conv_layout_kwargs(kv_cache_manager_cls, "qwen3_next")) kv_cache_manager = kv_cache_manager_cls( # mamba (KDA) cache parameters @@ -3162,7 +3073,10 @@ def _create_kv_cache_manager( num_layers=sum(full_attention_layer_mask), layer_mask=full_attention_layer_mask, num_kv_heads=1, - head_dim=config.kv_lora_rank + config.qk_rope_head_dim, + head_dim=(int(text_config.kv_lora_rank) + + int(getattr(text_config, "qk_rope_head_dim", 0) or 0) + if is_glm else config.kv_lora_rank + + config.qk_rope_head_dim), tokens_per_block=tokens_per_block, max_seq_len=max_seq_len, max_num_tokens=max_num_tokens, @@ -3173,7 +3087,7 @@ def _create_kv_cache_manager( spec_config=spec_config, is_estimating_kv_cache=estimating_kv_cache, execution_stream=execution_stream, - **kimi_extra_kwargs, + **kda_extra_kwargs, **manager_extra_kwargs, ) elif is_mla(config): diff --git a/tensorrt_llm/_torch/pyexecutor/config_utils.py b/tensorrt_llm/_torch/pyexecutor/config_utils.py index f8b9fdc83324..37ec53db3fe7 100644 --- a/tensorrt_llm/_torch/pyexecutor/config_utils.py +++ b/tensorrt_llm/_torch/pyexecutor/config_utils.py @@ -207,13 +207,8 @@ def is_glm5_next(config: transformers.PretrainedConfig) -> bool: the composite VLM config (model_type "glm5_next" with a nested text_config). """ - model_type = getattr(config, "model_type", None) - if model_type == "glm5_next_text": - return getattr(config, "layer_types", None) is not None - if model_type == "glm5_next": - text_config = getattr(config, "text_config", None) - return text_config is not None and is_glm5_next(text_config) - return False + return getattr(config, "model_type", + None) in ("glm5_next", "glm5_next_text") def unwrap_glm5_next_text_config( @@ -242,7 +237,8 @@ def get_glm5_next_layer_masks( checkpoint. """ config = unwrap_glm5_next_text_config(config) - if len(config.layer_types) != config.num_hidden_layers: + if (getattr(config, "layer_types", None) is None + or len(config.layer_types) != config.num_hidden_layers): raise ValueError( "glm5_next layer_types must contain num_hidden_layers entries") full_mask, kda_mask = [], [] diff --git a/tests/unittest/_torch/modeling/test_glm5_next_contracts.py b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py index c71170b9df88..582df8738b2c 100644 --- a/tests/unittest/_torch/modeling/test_glm5_next_contracts.py +++ b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py @@ -9,15 +9,16 @@ import torch from transformers import PretrainedConfig -from tensorrt_llm._torch.distributed import AllReduce +from tensorrt_llm._torch.distributed import AllReduce, AllReduceStrategy from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.models.checkpoints.hf.glm5_next_weight_mapper import ( - Disposition, audit_glm5_next_checkpoint, ) from tensorrt_llm._torch.models.modeling_glm5_next import ( SPARSE_MLP, + Glm5NextAllReduce, Glm5NextDecoderLayer, + Glm5NextForCausalLM, Glm5NextLinearAttention, Glm5NextMTP, Glm5NextRuntimeContext, @@ -25,8 +26,11 @@ build_glm5_next_runtime_context, glm5_next_tp_reduces, ) -from tensorrt_llm._torch.models.modeling_glm5_next_vision import Glm5NextVisionModelBase -from tensorrt_llm._torch.pyexecutor.config_utils import get_glm5_next_layer_masks +from tensorrt_llm._torch.models.modeling_glm5_next_vision import ( + Glm5NextVisionModelBase, + Glm5NextVLM, +) +from tensorrt_llm._torch.pyexecutor.config_utils import get_glm5_next_layer_masks, is_glm5_next from tensorrt_llm.mapping import Mapping @@ -83,6 +87,60 @@ def _config(): return config +@pytest.mark.cpu_only +@pytest.mark.parametrize("override", [None, "NCCL", "TWOSHOT"]) +def test_allreduce_defaults_honor_user_strategy(override): + from tensorrt_llm.llmapi.llm_args import TorchLlmArgs + from tensorrt_llm.llmapi.llm_utils import apply_model_defaults_to_llm_args + + args = TorchLlmArgs( + model="/tmp/dummy_model", **({"allreduce_strategy": override} if override else {}) + ) + apply_model_defaults_to_llm_args(args, Glm5NextVLM.get_model_defaults(args)) + assert args.allreduce_strategy == (override or "ONESHOT") + strategy = AllReduceStrategy[args.allreduce_strategy] + with patch( + "tensorrt_llm._torch.distributed.AllReduce", side_effect=lambda **kw: Mock() + ) as create: + reduction = Glm5NextAllReduce(Mapping(world_size=4, tp_size=4), strategy=strategy) + assert create.call_args_list[0].kwargs["strategy"] == strategy + reduction(torch.zeros(512, 8)) + reduction(torch.zeros(513, 8)) + if override: + assert reduction.small is reduction.large + assert reduction.small.call_count == 2 + else: + assert create.call_args_list[1].kwargs["strategy"] == AllReduceStrategy.NCCL + reduction.small.assert_called_once() + reduction.large.assert_called_once() + + +@pytest.mark.cpu_only +def test_mtp_rejects_unsupported_attention_dp_lm_head_sharding(): + from tensorrt_llm.llmapi import MTPDecodingConfig + + mapping = Mapping( + world_size=4, tp_size=4, enable_attention_dp=True, enable_lm_head_tp_in_adp=True + ) + with pytest.raises(NotImplementedError, match="tensor-parallel LM head"): + Glm5NextForCausalLM( + ModelConfig( + pretrained_config=_config(), + mapping=mapping, + spec_config=MTPDecodingConfig(max_draft_len=3), + ) + ) + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("model_type", ["glm5_next", "glm5_next_text"]) +def test_model_type_recognition_does_not_hide_missing_schedule(model_type): + config = SimpleNamespace(model_type=model_type, num_hidden_layers=1) + assert is_glm5_next(config) + with pytest.raises(ValueError, match="layer_types"): + get_glm5_next_layer_masks(config) + + @pytest.mark.cpu_only def test_layer_masks_accept_composite_and_text_configs(): config = _config() @@ -119,13 +177,13 @@ def test_runtime_context_uses_prepared_schedules_and_live_lengths(is_cuda_graph) context = build_glm5_next_runtime_context(metadata) assert context.ctx_cu_seqlens is prepared.glm_ctx_cu_seqlens assert context.cached_lens is prepared.glm_cached_lens_host - assert context.gen_phase == "verify" assert context.gen_tokens_per_request == 4 - torch.testing.assert_close(context.kv_lens, live_lengths.long()) + torch.testing.assert_close(context.kv_lens, live_lengths) + assert context.kv_lens.data_ptr() == live_lengths.data_ptr() # MTP rewinds the device lengths without changing the host prefill schedule. live_lengths[1] -= 2 torch.testing.assert_close( - build_glm5_next_runtime_context(metadata).kv_lens, torch.tensor([6, 7]) + build_glm5_next_runtime_context(metadata).kv_lens, torch.tensor([6, 7], dtype=torch.int32) ) prepared.glm_block_tables = None with pytest.raises(RuntimeError, match="requires prepared glm_block_tables"): @@ -148,8 +206,8 @@ def test_checkpoint_routes_vision_and_optional_mtp_separately(): ] plain = audit_glm5_next_checkpoint(keys, config) mtp = audit_glm5_next_checkpoint(keys, config, num_mtp_layers=1) - assert plain.disposition[keys[0]] == Disposition.IGNORED - assert plain.disposition[keys[2]] == Disposition.IGNORED + assert keys[0] in plain.ignored + assert keys[2] in plain.ignored assert mtp.destinations["model.layers.2.eh_proj.weight"] == keys[2] assert mtp.destinations["model.layers.0.self_attn.A_log"] == keys[1] assert mtp.unresolved == ["unexpected.weight"] @@ -187,16 +245,21 @@ def test_cache_manager_routing_guards(route, monkeypatch): @pytest.mark.cpu_only @pytest.mark.parametrize("fp8_kv_cache", [False, True], ids=["bf16-kv", "fp8-kv"]) -def test_kv_cache_dtype_reaches_manager_construction(fp8_kv_cache): +@pytest.mark.parametrize( + "verify_kernel", [None, True, False], ids=["no-mtp", "fused-mtp", "sequential-mtp"] +) +def test_kv_cache_dtype_reaches_manager_construction(fp8_kv_cache, verify_kernel): from tensorrt_llm._torch.attention.backends.sparse.glm_kpool import Glm5NextCacheManager from tensorrt_llm._torch.pyexecutor._util import _create_kv_cache_manager from tensorrt_llm.bindings import DataType - from tensorrt_llm.llmapi import KvCacheConfig + from tensorrt_llm.llmapi import KvCacheConfig, MTPDecodingConfig from tensorrt_llm.models.modeling_utils import QuantConfig from tensorrt_llm.quantization import QuantAlgo + spec_config = MTPDecodingConfig(max_draft_len=3) if verify_kernel is not None else None config = ModelConfig( pretrained_config=_config().text_config, + spec_config=spec_config, quant_config=QuantConfig( quant_algo=QuantAlgo.FP8_BLOCK_SCALES, kv_cache_quant_algo=QuantAlgo.FP8 if fp8_kv_cache else None, @@ -206,11 +269,20 @@ def test_kv_cache_dtype_reaches_manager_construction(fp8_kv_cache): class AllocationReached(Exception): pass - with patch.object(Glm5NextCacheManager, "__new__", side_effect=AllocationReached) as allocate: + allocate = Mock(side_effect=AllocationReached) + + class RecordingManager(Glm5NextCacheManager): + def __init__(self, *args, **kwargs): + allocate(*args, **kwargs) + + with patch( + "tensorrt_llm._torch.modules.kimi_kda._kda_kernels.is_kda_mtp_verify_available", + return_value=verify_kernel is True, + ): with pytest.raises(AllocationReached): _create_kv_cache_manager( model_engine=None, - kv_cache_manager_cls=Glm5NextCacheManager, + kv_cache_manager_cls=RecordingManager, model_config=config, mapping=Mapping(), kv_cache_config=KvCacheConfig( @@ -219,7 +291,7 @@ class AllocationReached(Exception): tokens_per_block=32, max_seq_len=128, max_batch_size=1, - spec_config=None, + spec_config=spec_config, sparse_attention_config=None, max_num_tokens=64, max_beam_width=1, @@ -227,10 +299,12 @@ class AllocationReached(Exception): dtype=torch.bfloat16, is_draft=False, ) - allocate.assert_called_once() - assert allocate.call_args.kwargs["dtype"] == ( - DataType.FP8 if fp8_kv_cache else DataType.BF16 - ) + allocate.assert_called_once() + assert allocate.call_args.kwargs["dtype"] == (DataType.FP8 if fp8_kv_cache else DataType.BF16) + if verify_kernel is True: + assert allocate.call_args.kwargs["kda_replay_num_spec"] == 3 + else: + assert "kda_replay_num_spec" not in allocate.call_args.kwargs @pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") @@ -271,6 +345,7 @@ def test_kda_shards_preserve_fp32_gate_parameters(): assert layer.A_log.dtype == layer.dt_bias.dtype == torch.float32 biases.append(layer.shard_checkpoint_tensor("dt_bias", full_bias)) logs.append(layer.shard_checkpoint_tensor("A_log", full_log)) + assert "use_full_rank_gate" not in config.linear_attn_config torch.testing.assert_close(torch.cat(biases), full_bias) torch.testing.assert_close(torch.cat(logs), full_log) @@ -440,9 +515,8 @@ def test_mixed_batches_keep_one_full_batch_moe_call( ) hidden = torch.arange(counts[rank] * 4 * 8, dtype=torch.float32).view(-1, 4, 8) reduction = Mock(side_effect=lambda value: value) - attention = SimpleNamespace( - tp_all_reduce=reduction if glm5_next_tp_reduces(mapping) else None - ) + attention = torch.nn.Module() + attention.tp_all_reduce = reduction if glm5_next_tp_reduces(mapping) else None def attend(x, *args, reduce=True, **kwargs): return attention.tp_all_reduce(x) if reduce and attention.tp_all_reduce else x @@ -456,7 +530,7 @@ def attend(x, *args, reduce=True, **kwargs): f"forward_{phase}", create_autospec(implementation, side_effect=attend), ) - attention.forward_mixed = MethodType(Glm5NextSparseAttention.forward_mixed, attention) + attention.forward = MethodType(Glm5NextSparseAttention.forward, attention) connection = SimpleNamespace( pre_mapping=lambda streams: (None, None, streams.mean(dim=1)), post_mapping=lambda output, residual, post, comb: residual + output.unsqueeze(1), @@ -474,7 +548,6 @@ def attend(x, *args, reduce=True, **kwargs): ) if layer_kind == "decoder": attn_input = hidden.mean(dim=1) - layer.forward_direct = MethodType(Glm5NextDecoderLayer.forward_direct, layer) layer.run_mlp = MethodType(Glm5NextDecoderLayer.run_mlp, layer) result = Glm5NextDecoderLayer.forward(layer, hidden_states=hidden, runtime_ctx=context) expected = hidden + attn_input.unsqueeze(1) @@ -514,7 +587,9 @@ def attend(x, *args, reduce=True, **kwargs): ) else: attention.forward_prefill.assert_not_called() - generation = getattr(attention, f"forward_{context.gen_phase}") + generation = ( + attention.forward_decode if tokens_per_request == 1 else attention.forward_verify + ) if generations: generation.assert_called_once() assert generation.call_args.kwargs["metadata"] is metadata @@ -532,7 +607,7 @@ def attend(x, *args, reduce=True, **kwargs): @pytest.mark.cpu_only @pytest.mark.parametrize("generation_phase", ["decode", "verify"]) @pytest.mark.parametrize("reduce_output", [False, True], ids=["attention-dp", "tp"]) -def test_mixed_attention_splits_only_attention_and_reduces_once(generation_phase, reduce_output): +def test_packed_attention_splits_phases_and_reduces_once(generation_phase, reduce_output): gen_tokens = 4 if generation_phase == "verify" else 1 hidden = torch.randn(3 + gen_tokens, 8) ctx = torch.zeros(3, 8) @@ -544,14 +619,18 @@ def test_mixed_attention_splits_only_attention_and_reduces_once(generation_phase forward_verify=Mock(return_value=gen), tp_all_reduce=reduction, ) - result = Glm5NextSparseAttention.forward_mixed( - attention, - hidden, + metadata = SimpleNamespace() + context = Glm5NextRuntimeContext( + num_contexts=1, num_ctx_tokens=3, - prefill={"marker": "context"}, - generation={"marker": "generation"}, - generation_phase=generation_phase, + num_generations=1, + ctx_cu_seqlens=[0, 3], + cached_lens=[0, 8], + kv_lens=torch.tensor([3, 8 + gen_tokens]), + metadata=metadata, + gen_tokens_per_request=gen_tokens, ) + result = Glm5NextSparseAttention.forward(attention, hidden, metadata, runtime_ctx=context) expected = torch.cat([ctx, gen]) + (10 if reduce_output else 0) torch.testing.assert_close(result, expected) attention.forward_prefill.assert_called_once() From 34a770635d64f55afb9a37ec51c25b577fd8091e Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Thu, 17 Sep 2026 01:55:38 -0700 Subject: [PATCH 11/35] [None][perf] Enable compact KDA decode for 64 heads at small batches Signed-off-by: Ruocheng Jia --- cpp/tensorrt_llm/kernels/kdaDecode/kdaDecode.h | 3 ++- .../_torch/modules/kimi_kda/test_kda_decode_op.py | 12 ++++++++---- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/cpp/tensorrt_llm/kernels/kdaDecode/kdaDecode.h b/cpp/tensorrt_llm/kernels/kdaDecode/kdaDecode.h index b5840cc34e7f..fa9c6770eb2b 100644 --- a/cpp/tensorrt_llm/kernels/kdaDecode/kdaDecode.h +++ b/cpp/tensorrt_llm/kernels/kdaDecode/kdaDecode.h @@ -31,7 +31,8 @@ constexpr int kCompactHeadsWorkThreshold = 144; constexpr bool isSupportedHeadCount(int numHeads) { return numHeads == 1 || numHeads == 2 || numHeads == 3 || numHeads == 4 || numHeads == 6 || numHeads == 8 - || numHeads == 12 || numHeads == 16 || numHeads == 24 || numHeads == 32 || numHeads == 48 || numHeads == 96; + || numHeads == 12 || numHeads == 16 || numHeads == 24 || numHeads == 32 || numHeads == 48 || numHeads == 64 + || numHeads == 96; } //! Select the compact-head kernel within the measured KDA decode work threshold. diff --git a/tests/unittest/_torch/modules/kimi_kda/test_kda_decode_op.py b/tests/unittest/_torch/modules/kimi_kda/test_kda_decode_op.py index df46ae1e4c84..d09618ab16b7 100644 --- a/tests/unittest/_torch/modules/kimi_kda/test_kda_decode_op.py +++ b/tests/unittest/_torch/modules/kimi_kda/test_kda_decode_op.py @@ -596,18 +596,22 @@ def clone_args(conv_pool: torch.Tensor) -> dict: @torch.no_grad() @pytest.mark.parametrize( - ("num_heads", "indexed_state"), + ("num_heads", "batch_size", "indexed_state"), [ - (2, False), - (96, True), + (2, 1, False), + (96, 1, True), + (64, 1, False), + (64, 2, True), + (64, 3, True), ], ) def test_kda_decode_is_cuda_graph_safe( num_heads: int, + batch_size: int, indexed_state: bool, ) -> None: args = _make_direct_decode_args( - 1, + batch_size, num_heads, indexed_state=indexed_state, ) From 971d447c5072e41c70079c4866b52b93573f9613 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Thu, 17 Sep 2026 02:05:21 -0700 Subject: [PATCH 12/35] [None][docs] Simplify GLM-5.3-Flash supported-model note Signed-off-by: Ruocheng Jia --- docs/source/models/supported-models.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/models/supported-models.md b/docs/source/models/supported-models.md index d644a2007ee4..492a33d7d2e4 100644 --- a/docs/source/models/supported-models.md +++ b/docs/source/models/supported-models.md @@ -96,7 +96,7 @@ statuses for the same architecture in the two matrices. [^17]: Kimi K3 has no MTP or EAGLE-3 head, and its DSpark checkpoints are not compatible with plain `DFlash`. [^18]: NGram and standalone Suffix Automaton (SA) use model-free drafting on the PyTorch backend, so they are not listed in individual entries. This does not imply universal end-to-end support: compatibility depends on each model's multi-token verification and cache-management paths and may be untested or explicitly restricted. [^19]: KV cache reuse for hybrid recurrent-attention models requires an explicit recurrent-state snapshot policy, such as `kv_cache_config.mamba_state_config.periodic_snapshot_interval`; the model default disables reuse when no snapshot policy is configured. -[^21]: Supports text, image and video inputs and one-model MTP with 1 to 5 draft tokens. Requires `transformers==5.17.0` as documented in the deployment guide, whose `Glm5NextProcessor` performs the image and video preprocessing. Supports FP8 KV cache and attention data parallelism with MTP. Beam search is not supported; disaggregated serving requires the Python NIXL transceiver; KV cache block reuse requires `kv_cache_config.mamba_state_config.periodic_snapshot_interval`. See the [deployment guide](../deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md). +[^21]: Supports text, image, and video inputs, MTP (including with attention data parallelism), and FP8 KV cache. Requires `transformers==5.17.0`. Beam search is not supported. See the [GLM-5.3-Flash deployment guide](../deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md) for setup and feature-specific requirements. # Encoder-Decoder Feature Support Matrix (PyTorch Backend) From e18a345f2c2e70a4428dffc28969c18575fb29ac Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Fri, 18 Sep 2026 02:48:43 -0700 Subject: [PATCH 13/35] [None][fix] Honor GLM indexer head-weight strides Pass row and head strides into both k-pool scoring branches without copying fused projection views. Cover strided weights and top-512 selection with more than 512 complete pools. Validation: 12 CPU Triton interpreter cases, offline SM100 compilation, and applicable pre-commit hooks pass. Native GPU execution and full-model accuracy validation remain pending GPU availability. Signed-off-by: Ruocheng Jia --- .../backends/sparse/glm_kpool/kernels.py | 14 ++++- .../sparse/glm_kpool/test_kernels.py | 56 +++++++++++++++++++ 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py index 2eb0bf20f70d..1d6baa06553e 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py @@ -191,6 +191,8 @@ def _kpool_score_kernel( REQ, OUT, num_rows, + w_row_stride, + w_head_stride, slot_stride, row_stride, bt_stride, @@ -268,7 +270,9 @@ def _kpool_score_kernel( ).to(tl.float32) scores = tl.dot(q, tl.trans(keys), input_precision=PRECISION) # [HP, BP] scores = tl.maximum(scores * q_scale, 0.0) - w = tl.load(W + r * H + h, mask=hmask & in_range, other=0.0).to(tl.float32) + w = tl.load( + W + r * w_row_stride + h * w_head_stride, mask=hmask & in_range, other=0.0 + ).to(tl.float32) mixed = tl.sum(scores * (w * w_scale)[:, None], axis=0) mixed = tl.where(valid, mixed, min_value) tl.store(OUT + r * num_pools_max + j, mixed, mask=(j < num_pools_max) & in_range) @@ -291,7 +295,9 @@ def _kpool_score_kernel( ).to(tl.float32) scores = tl.dot(q, tl.trans(keys), input_precision=PRECISION) # [HP, BP] scores = tl.maximum(scores * q_scale, 0.0) - w = tl.load(W + r * H + h, mask=hmask & in_range, other=0.0).to(tl.float32) + w = tl.load( + W + r * w_row_stride + h * w_head_stride, mask=hmask & in_range, other=0.0 + ).to(tl.float32) mixed = tl.sum(scores * (w * w_scale)[:, None], axis=0) mixed = tl.where(valid, mixed, min_value) tl.store(OUT + r * num_pools_max + j, mixed, mask=(j < num_pools_max) & in_range) @@ -317,7 +323,7 @@ def kpool_score( """Pool scores ``[N, num_pools_max]`` fp32. ``q`` is ``[N, H, head_dim]`` (bf16 or fp32, contiguous), ``weights`` - ``[N, H]``, ``kv_lens[i]`` row ``i``'s visible length. Only the + ``[N, H]`` (possibly strided), ``kv_lens[i]`` row ``i``'s visible length. Only the ``kv_len // kpool`` complete pools of each row are scored; the rest hold the fp32 minimum. ``rows_per_program > 1`` requires rows that share a block table to be consecutive with non-decreasing ``kv_lens``: either all @@ -348,6 +354,8 @@ def kpool_score( kv_lens if request_ids is None else request_ids, out, n, + weights.stride(0), + weights.stride(1), index_pool.stride(0), index_pool.stride(1), block_tables.stride(0), diff --git a/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py b/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py index 441044ff1fd4..2a57f034958d 100644 --- a/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py +++ b/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py @@ -152,6 +152,62 @@ def test_pool_scores_generation_rows(precision): torch.testing.assert_close(got[visible], want[visible], atol=2e-3, rtol=2e-3) +@pytest.mark.parametrize("layout", ["generation", "prefill_shared", "prefill_boundary"]) +@pytest.mark.parametrize("weight_head_stride", [1, 2]) +def test_pool_scores_strided_head_weights(layout, weight_head_stride): + gen = torch.Generator(device="cuda").manual_seed(23) + n, heads, num_pools_max = 4, 32, 520 + pages = num_pools_max * KPOOL // TPB + num_tables = n if layout == "generation" else 2 if layout == "prefill_boundary" else 1 + slots = num_tables * pages + 8 + pool = _strided_index_pool(slots, gen) + tables = _paged_tables(num_tables, pages, slots, gen) + request_ids = None + if layout != "generation": + request_ids = torch.tensor( + [0, 0, 1, 1] if layout == "prefill_boundary" else [0, 0, 0, 0], + device="cuda", + dtype=torch.int32, + ) + q = torch.randn(n, heads, HD, generator=gen, device="cuda").to(torch.bfloat16) + # Match project_state's [key | gate | head weights] slice: [T, 32] + # with row stride 288. Also exercise a non-unit head stride. + projection = torch.rand( + n, 2 * HD + heads * weight_head_stride, generator=gen, device="cuda" + ).to(torch.bfloat16) + weights = projection[:, 2 * HD :: weight_head_stride] + assert weights.stride() == (2 * HD + heads * weight_head_stride, weight_head_stride) + assert not weights.is_contiguous() + # More than 512 complete pools forces top-k to discard candidates. + kv_lens = torch.tensor([2052, 2053, 2056, 2057], device="cuda", dtype=torch.int64) + kwargs = dict(num_pools_max=num_pools_max, q_scale=HD**-0.5, w_scale=heads**-0.5) + got = kpool_score( + q, + weights, + pool, + tables, + kv_lens, + TPB, + head_dim=HD, + kpool=KPOOL, + rows_per_program=1 if request_ids is None else 16, + request_ids=request_ids, + precision="tf32", + **kwargs, + ) + reference_tables = tables if request_ids is None else tables[request_ids.long()] + want = _reference_scores(q, weights, pool, reference_tables, kv_lens, **kwargs) + visible = want > FP32_MIN + assert torch.equal(got <= FP32_MIN, ~visible) + torch.testing.assert_close(got[visible], want[visible], atol=2e-3, rtol=2e-3) + torch.testing.assert_close( + got.topk(512, dim=-1).indices.sort(dim=-1).values, + want.topk(512, dim=-1).indices.sort(dim=-1).values, + atol=0, + rtol=0, + ) + + def test_pool_scores_packed_context_rows_share_tables(): """rows_per_program=16 with request_ids: the packed query tokens of several requests, in position order, including a group straddling two From d76547422b6e1feb910dc436e50d32d211509ca1 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Fri, 18 Sep 2026 02:49:00 -0700 Subject: [PATCH 14/35] [None][docs] Trim GLM-5.3-Flash benchmark commentary Keep the reported performance figures and workload-dependent MTP behavior, while leaving the detailed TTFT investigation in the experiment records. This does not claim the observed TTFT variability is resolved. Signed-off-by: Ruocheng Jia --- .../deployment-guide-for-glm-5.3-flash-on-trtllm.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md index 0b5d8f6e7b35..8b07ace3fdb0 100644 --- a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md @@ -451,4 +451,4 @@ The horizontal axis is `1000 / mean_tpot_ms`, excluding TTFT. The vertical axis ![GLM-5.3-Flash FP8 performance on 4x B200](../media/glm_5_3_flash_fp8_perf.png) -With TP4 / EP4, MTP3 improves single-user decode speed from approximately 153 to 372 tok/s/user. At concurrency 128, the four configurations deliver approximately 5.9K–6.3K output tok/s in aggregate. Attention DP4 / EP4 was also repeated in ascending and descending concurrency order; earlier TTFT spikes at concurrency 8 and 32 did not consistently recur, and all matching samples remain included. MTP uses natural acceptance; random-token workloads can have different acceptance rates from real conversations. This sweep covers the plotted concurrency range, not peak throughput or maximum-context validation. +With TP4 / EP4, MTP3 improves single-user decode speed from approximately 153 to 372 tok/s/user. At concurrency 128, the four configurations deliver approximately 5.9K–6.3K output tok/s in aggregate. MTP acceptance and speedup depend on the workload; these measurements use random-token prompts. From a02fbb621cfb06bdae8c5b95ac3a853cda533822 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Sun, 20 Sep 2026 09:44:38 -0700 Subject: [PATCH 15/35] [TRTLLM-16480][perf] Extend mHC pre-mapping tuning coverage Generate and map matching 1024-token tuning buckets above 8192 through the rounded warmup limit, avoiding FMA cache-miss fallbacks for long prefill. Preserve the existing fused-HC policy and cover aligned and non-aligned warmup boundaries with a CPU regression test. Signed-off-by: Ruocheng Jia --- tensorrt_llm/_torch/modules/mhc/mhc_cuda.py | 36 +++++++++++++-------- tests/unittest/_torch/modules/test_mhc.py | 24 ++++++++++++++ 2 files changed, 47 insertions(+), 13 deletions(-) diff --git a/tensorrt_llm/_torch/modules/mhc/mhc_cuda.py b/tensorrt_llm/_torch/modules/mhc/mhc_cuda.py index 8052199b14ef..3efeff5f409a 100644 --- a/tensorrt_llm/_torch/modules/mhc/mhc_cuda.py +++ b/tensorrt_llm/_torch/modules/mhc/mhc_cuda.py @@ -17,7 +17,7 @@ Falls back to FMA when DeepGEMM is unavailable or autotuner cache misses. """ -from functools import lru_cache +from functools import lru_cache, partial from typing import Any, List import torch @@ -211,28 +211,34 @@ def mhc_gemm_rms_fma_cuda( # --------------------------------------------------------------------------- -def _mhc_gen_tuning_buckets(x: int): - """Generate M-dimension tuning buckets for MHC pre_mapping. +_MHC_FINE_GRAINED_TUNING_LIMIT = 8192 +_MHC_COARSE_TUNING_STEP = 1024 - Buckets: 1, 2, 4, 8, 16, 32, 64, 128, 256, 384, 512, 768, 1024, ... - Small M uses powers-of-2 for fine granularity; large M uses 128 steps. - """ + +def _mhc_gen_tuning_buckets(x: int, *, large_m_step: int | None = None) -> tuple[int, ...]: + """Generate fine-grained buckets, optionally extending above 8192 with coarser steps.""" + if large_m_step is not None: + # Cover the rounded-up runtime bucket even for non-aligned warmup sizes. + x = _mhc_map_to_tuning_bucket(x, large_m_step=large_m_step) buckets = (1, 2, 4, 8, 16, 32, 64, 128) if x >= 128: - x = min(x, 8192) - x = max(x, 1024) - buckets += tuple(range(256, x + 1, 128)) + fine_upper = max(1024, min(x, _MHC_FINE_GRAINED_TUNING_LIMIT)) + buckets += tuple(range(256, fine_upper + 1, 128)) + if large_m_step is not None and x > _MHC_FINE_GRAINED_TUNING_LIMIT: + coarse_start = (_MHC_FINE_GRAINED_TUNING_LIMIT // large_m_step + 1) * large_m_step + buckets += tuple(range(coarse_start, x + 1, large_m_step)) return buckets -def _mhc_map_to_tuning_bucket(x: int) -> int: +def _mhc_map_to_tuning_bucket(x: int, *, large_m_step: int | None = None) -> int: """Map an inference-time M to the nearest tuning bucket (round up).""" if x <= 128: v = 1 while v < x: v *= 2 return min(v, 128) - return ((x + 127) // 128) * 128 + step = large_m_step if large_m_step is not None and x > _MHC_FINE_GRAINED_TUNING_LIMIT else 128 + return ((x + step - 1) // step) * step _FMA_TILE_N_OPTIONS = (1, 2, 3, 4, 6, 8, 12, 24) @@ -263,8 +269,12 @@ class MhcPreMappingRunner(TunableRunner): DynamicTensorSpec( input_idx=0, dim_idx=0, - gen_tuning_buckets=_mhc_gen_tuning_buckets, - map_to_tuning_buckets=_mhc_map_to_tuning_bucket, + gen_tuning_buckets=partial( + _mhc_gen_tuning_buckets, large_m_step=_MHC_COARSE_TUNING_STEP + ), + map_to_tuning_buckets=partial( + _mhc_map_to_tuning_bucket, large_m_step=_MHC_COARSE_TUNING_STEP + ), ), ), # residual (input[2]) dim 0 = M, same as x (input[0]) dim 0 diff --git a/tests/unittest/_torch/modules/test_mhc.py b/tests/unittest/_torch/modules/test_mhc.py index 2bfd5202ef77..0ef432e33e50 100644 --- a/tests/unittest/_torch/modules/test_mhc.py +++ b/tests/unittest/_torch/modules/test_mhc.py @@ -295,6 +295,30 @@ def generate_head_data( # --------------------------------------------------------------------------- +@pytest.mark.cpu_only +def test_mhc_pre_mapping_tuning_bucket_coverage() -> None: + from tensorrt_llm._torch.modules.mhc.mhc_cuda import MhcPreMappingRunner + + spec = MhcPreMappingRunner.tuning_config.dynamic_tensor_specs[0] + for max_tokens in (1024, 1025, 8191, 8192, 8193, 13000, 13312, 16384, 32768): + tuned_buckets = set(spec.gen_tuning_buckets(max_tokens)) | {max_tokens} + for num_tokens in range(1, max_tokens + 1): + bucket = spec.map_to_tuning_buckets(num_tokens) + step = 128 if num_tokens <= 8192 else 1024 + assert num_tokens <= bucket < num_tokens + step + assert bucket in tuned_buckets + assert tuple(bucket for bucket in spec.gen_tuning_buckets(16384) if bucket > 8192) == ( + 9216, + 10240, + 11264, + 12288, + 13312, + 14336, + 15360, + 16384, + ) + + @pytest.mark.parametrize("n", [1, 32, 64, 128, 256, 512, 4096, 8192]) @pytest.mark.parametrize("hidden_size", [4096]) @pytest.mark.parametrize("hc_mult", [4]) From f0482609e0bc44d5f79e14ce5eddbadbde5bcf8a Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Sun, 20 Sep 2026 20:59:03 -0700 Subject: [PATCH 16/35] [TRTLLM-16480][perf] Optimize GLM-5.3-Flash small-batch decoding Overlap shared and routed experts under TP CUDA graphs, use native 16-head BF16 sparse MLA for small generation batches, and specialize the FP32 router projection. Preserve existing prefill and FP8 KV attention paths. Validation: 65 CPU and 27 GPU checks passed, with full GSM8K paired accuracy and MTP3 accuracy/acceptance-length checks. Three-run B200 serving measurements preserve C8 throughput and match local vLLM TP4/EP4 C1 TPOT. Signed-off-by: Ruocheng Jia --- .../backends/sparse/glm_kpool/backend.py | 30 ++++++- .../backends/sparse/glm_kpool/kernels.py | 48 +++++++++++ .../sparse/glm_kpool/native_decode.py | 86 +++++++++++++++++++ .../_torch/models/modeling_glm5_next.py | 67 ++++++++++++--- .../_torch/moe/fused_moe/fp32_router_gemm.py | 57 ++++++++++++ .../sparse/glm_kpool/test_kernels.py | 54 ++++++++++++ .../_torch/moe/test_fp32_router_gemm.py | 30 +++++++ 7 files changed, 356 insertions(+), 16 deletions(-) create mode 100644 tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/native_decode.py create mode 100644 tensorrt_llm/_torch/moe/fused_moe/fp32_router_gemm.py create mode 100644 tests/unittest/_torch/moe/test_fp32_router_gemm.py diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py index 36e78e85dfba..b947b7438e8d 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py @@ -15,11 +15,12 @@ """GLM-5.3-Flash pool-compressed sparse MLA in the TRTLLM backend family. The model projects queries and selects pools; this backend owns paged latent -and indexer state, pool-key updates, and FlashMLA attention. Selection arrives +and indexer state, pool-key updates, and sparse attention kernels. Selection arrives as global latent-cache row IDs in GlmKpoolBackendForwardArgs.topk_rows. Queries have no rotary component and are absorbed into the 512-wide latent -space. FlashMLA consumes BF16 rows; FP8 cache rows are gathered and dequantized +space. Small TP generation batches use native TRTLLM-GEN query heads; other +shapes use FlashMLA. FP8 cache rows are gathered and dequantized in bounded query chunks. Prepared GLM page tables and live TRTLLM lengths provide the same cache contract for prefill, decode and verification. """ @@ -31,6 +32,8 @@ import torch +from tensorrt_llm._utils import get_sm_version + from ...interface import ( AttentionForwardArgs, AttentionInputType, @@ -40,6 +43,7 @@ ) from ...trtllm import TrtllmAttention, TrtllmAttentionMetadata from .kernels import gather_fp8_kv_rows, kpool_expand, kpool_score, kpool_update +from .native_decode import GlmKpoolNativeDecode from .params import INDEX_SENTINEL, GlmKpoolSparseParams @@ -211,6 +215,7 @@ def __init__( #: ``qk_nope_head_dim``; absorption reassociates the matmuls but the #: score scale is unchanged. self.softmax_scale = float(sparse_params.qk_nope_head_dim) ** -0.5 + self._native_decode = GlmKpoolNativeDecode() @classmethod def support_fused_rope(cls) -> bool: @@ -613,4 +618,23 @@ def forward( raise ValueError(f"glm_kpool forward is phase-explicit, got {input_type!r}") state = self._cache_state(metadata) kv_rows, _, _ = latent_pool_rows(state.latent_pool) - return self._finalize_output(self._dispatch_sparse_core(q, kv_rows, topk_rows), output) + native_supported = ( + q.is_cuda + and q.dtype == kv_rows.dtype == torch.bfloat16 + and q.shape[1:] == (16, 512) + and kv_rows.shape[0] % 32 == 0 + and get_sm_version() in (100, 103) + ) + if native_supported: + # Reserve during prefill/profiling too, before sizing the KV pool. + # Scratch is shared through metadata; each backend owns only counters. + self._native_decode.prepare_workspace(q, metadata) + if ( + native_supported + and input_type == AttentionInputType.generation_only + and 1 <= q.shape[0] <= 8 + ): + out = self._native_decode(q, kv_rows, topk_rows, self.softmax_scale, metadata) + else: + out = self._dispatch_sparse_core(q, kv_rows, topk_rows) + return self._finalize_output(out, output) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py index 1d6baa06553e..9b4020b0f895 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py @@ -494,3 +494,51 @@ def kpool_expand( num_warps=4, ) return out + + +@triton.jit +def _compact_sparse_rows_kernel( + Indices, + Packed, + Lengths, + Nonempty, + Stride: tl.constexpr, + Width: tl.constexpr, + OutWidth: tl.constexpr, + Block: tl.constexpr, +): + row = tl.program_id(0) + col = tl.arange(0, Block) + value = tl.load(Indices + row * Stride + col, col < Width, -1) + valid = (col < Width) & (value >= 0) + destination = tl.cumsum(valid.to(tl.int32)) - 1 + length = tl.sum(valid.to(tl.int32)) + tl.store(Packed + row * OutWidth + destination, value, valid) + if length == 0: + tl.store(Packed + row * OutWidth, 0) + tl.store(Lengths + row, tl.maximum(length, 1)) + tl.store(Nonempty + row, length > 0) + + +def compact_sparse_rows(indices: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + """Pack nonnegative physical row IDs, preserving order and duplicates. + + Empty rows get one dummy ID and are identified by the returned nonempty mask. + The padded tail remains -1, and all outputs have capture-stable shapes. + """ + rows = indices.shape[0] + width = triton.cdiv(indices.shape[1], 64) * 64 + packed = torch.full((rows, width), -1, dtype=torch.int32, device=indices.device) + lengths = torch.empty(rows, dtype=torch.int32, device=indices.device) + nonempty = torch.empty(rows, dtype=torch.bool, device=indices.device) + _compact_sparse_rows_kernel[(rows,)]( + indices, + packed, + lengths, + nonempty, + indices.stride(0), + indices.shape[1], + width, + triton.next_power_of_2(indices.shape[1]), + ) + return packed, lengths, nonempty diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/native_decode.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/native_decode.py new file mode 100644 index 000000000000..794d28502e79 --- /dev/null +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/native_decode.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Small-batch BF16 sparse MLA with native TP-local query heads.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +import torch + +from .kernels import compact_sparse_rows + +if TYPE_CHECKING: + from ...trtllm import TrtllmAttentionMetadata + + +class GlmKpoolNativeDecode: + """Use shared attention scratch and backend-owned graph-stable counters. + + GLM selects physical latent row IDs, including holes for unavailable pools. + TRTLLM-GEN consumes these as sparse indices into a 32-row paged view, with + each query's valid selections packed before its length. No KV copy is made. + """ + + def __init__(self) -> None: + self._counter: torch.Tensor | None = None + self._workspace_size: int | None = None + + def prepare_workspace(self, q: torch.Tensor, metadata: TrtllmAttentionMetadata) -> None: + from flashinfer.utils import get_trtllm_gen_multi_ctas_kv_counter_bytes + + from ...fmha.flashinfer_trtllm_gen import _get_generation_workspace_layout + + capturing = torch.cuda.is_current_stream_capturing() + if self._workspace_size is None: + layout = _get_generation_workspace_layout(q.dtype, 1, 1, 16, 512, 1, 0) + self._workspace_size = int(layout["trtllm_gen_workspace_size"]) + workspace = metadata.effective_workspace + if workspace is None: + raise RuntimeError("glm_kpool native decode requires prepared attention workspace") + if workspace.numel() * workspace.element_size() < self._workspace_size: + if capturing: + raise RuntimeError("glm_kpool native decode workspace must be sized before capture") + workspace.resize_( + (self._workspace_size + workspace.element_size() - 1) // workspace.element_size() + ) + if self._counter is None: + if capturing: + raise RuntimeError( + "glm_kpool native decode counters must be allocated before capture" + ) + sm_count = torch.cuda.get_device_properties(q.device).multi_processor_count + size = get_trtllm_gen_multi_ctas_kv_counter_bytes(8, 16, sm_count) + # TRTLLM-GEN resets its counters after each invocation. + self._counter = torch.zeros(size, dtype=torch.uint8, device=q.device) + + def __call__( + self, + q: torch.Tensor, + kv_rows: torch.Tensor, + topk_rows: torch.Tensor, + scale: float, + metadata: TrtllmAttentionMetadata, + ) -> torch.Tensor: + from flashinfer.mla import trtllm_batch_decode_with_kv_cache_mla + + packed, lengths, nonempty = compact_sparse_rows(topk_rows) + output = trtllm_batch_decode_with_kv_cache_mla( + query=q.unsqueeze(1), + kv_cache=kv_rows.view(-1, 1, 32, 512), + workspace_buffer=metadata.effective_workspace.view(torch.uint8), + qk_nope_head_dim=256, + kv_lora_rank=512, + qk_rope_head_dim=0, + block_tables=packed.unsqueeze(1), + seq_lens=lengths, + max_seq_len=packed.shape[1], + sparse_mla_top_k=packed.shape[1], + bmm1_scale=scale, + backend="trtllm-gen", + sparse_mla_top_k_lens=lengths, + multi_ctas_kv_counter_buffer=self._counter, + ).squeeze(1) + # Empty (padded) requests use a dummy selection to keep the kernel's + # length positive; discard that output, matching FlashMLA's empty row. + return torch.where(nonempty[:, None, None], output, 0) diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next.py b/tensorrt_llm/_torch/models/modeling_glm5_next.py index 7075565ded57..03e0c8285968 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next.py @@ -53,8 +53,11 @@ from ..modules.kimi_kda.kimi_kda_mixer import KimiKDALinearAttention from ..modules.layer_norm import LayerNorm from ..modules.linear import Linear, TensorParallelMode +from ..modules.multi_stream_utils import maybe_execute_in_parallel from ..modules.rms_norm import RMSNorm +from ..moe.fused_moe.fp32_router_gemm import fp32_router_gemm from ..pyexecutor.config_utils import unwrap_glm5_next_text_config +from ..utils import AuxStreamType from .checkpoints.hf.glm5_next_weight_mapper import ( Glm5NextHfWeightMapper, glm5_next_is_quantized, @@ -1446,6 +1449,16 @@ def __init__(self, config: PretrainedConfig, moe_backend: str = "CUTLASS") -> No def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: flat = hidden_states.reshape(-1, self.hidden_size) + if ( + flat.is_cuda + and flat.dtype == torch.bfloat16 + and self.weight.dtype == torch.float32 + and self.weight.shape == (288, 4096) + and self.weight.is_contiguous() + and flat.stride(-1) == 1 + and 1 <= flat.shape[0] <= 8 + ): + return fp32_router_gemm(flat, self.weight) return torch.nn.functional.linear(flat.float(), self.weight) @@ -1463,6 +1476,7 @@ def __init__( model_config: ModelConfig, layer_idx: int, dtype: torch.dtype = torch.bfloat16, + aux_stream: torch.cuda.Stream | None = None, ) -> None: super().__init__() hidden = int(config.hidden_size) @@ -1521,6 +1535,9 @@ def __init__( # DeepSeek-V3 composition). self.mapping = model_config.mapping self.use_dp = bool(self.mapping.enable_attention_dp) + self.shared_expert_stream = None if self.use_dp else aux_stream + self.shared_expert_start = torch.cuda.Event() if self.shared_expert_stream else None + self.shared_expert_done = torch.cuda.Event() if self.shared_expert_stream else None # Reduce routed and TP-sharded shared-expert partials once. Under # attention DP the fused layer combines across ranks itself and the # shared expert is replicated, so there is nothing to reduce here. @@ -1558,16 +1575,26 @@ def forward( # serves prefill and decode with no host-dependent branching, so # decode stays CUDA-graph-capturable. ``all_rank_num_tokens`` drives # the fused layer's dispatch/combine under attention DP. - routed = self.experts( - flat, - self.gate(flat), - all_rank_num_tokens=all_rank_num_tokens if self.use_dp else None, + # Small TP batches leave enough GPU capacity to overlap the shared + # expert with routing and the routed experts. The helper limits stream + # switching to CUDA graphs and joins before the single reduction. + routed, shared = maybe_execute_in_parallel( + lambda: self.experts( + flat, + self.gate(flat), + all_rank_num_tokens=all_rank_num_tokens if self.use_dp else None, + ), + lambda: self.shared_experts(flat), + self.shared_expert_start, + self.shared_expert_done, + self.shared_expert_stream if flat.shape[0] <= 8 else None, + disable_on_compile=True, ) # Routed and shared are both rank partials (a K-dim partial per expert # in the TP4 layout, the local-expert partial sum in the TP4/EP4 # layout). Sum them, then exactly one reduction covers the whole MoE # branch -- the DeepSeek-V3 order. - mixed = routed + self.shared_experts(flat) + mixed = routed + shared if self.moe_all_reduce is not None: mixed = self.moe_all_reduce(mixed) return mixed.view_as(x) @@ -1638,6 +1665,7 @@ def __init__( schedule: Glm5NextSchedule, model_config: ModelConfig, dtype: torch.dtype = torch.bfloat16, + aux_stream: torch.cuda.Stream | None = None, ) -> None: super().__init__() self.layer_idx = layer_idx @@ -1663,7 +1691,7 @@ def __init__( model_config=model_config, ) self.mlp = ( - Glm5NextMoE(config, model_config, layer_idx, dtype=dtype) + Glm5NextMoE(config, model_config, layer_idx, dtype=dtype, aux_stream=aux_stream) if self.mlp_type == SPARSE_MLP else GatedMLP( hidden_size=int(config.hidden_size), @@ -1751,20 +1779,28 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: self.hc_mult = int(config.hc_mult) dtype = getattr(config, "torch_dtype", None) or torch.bfloat16 + # Decoder and draft layers execute serially and share this side stream. + self.aux_stream_dict = {} + if not model_config.mapping.enable_attention_dp and torch.cuda.is_available(): + self.aux_stream_dict[AuxStreamType.MoeShared] = torch.cuda.Stream() + self.embed_tokens = Embedding(int(config.vocab_size), int(config.hidden_size), dtype=dtype) self.layers = nn.ModuleList( [ - Glm5NextDecoderLayer(config, i, schedule, model_config, dtype=dtype) + Glm5NextDecoderLayer( + config, + i, + schedule, + model_config, + dtype=dtype, + aux_stream=self.aux_stream_dict.get(AuxStreamType.MoeShared), + ) for i in range(schedule.num_layers) ] ) self.norm = RMSNorm( hidden_size=int(config.hidden_size), eps=float(config.rms_norm_eps), dtype=dtype ) - # Read by the one-model MTP drafter factory (``MTPForCausalLM`` passes - # ``model.aux_stream_dict`` to every MTP layer). This model runs its - # branches on the main stream, so the draft layer receives an empty map. - self.aux_stream_dict: dict[Any, Any] = {} def forward( self, @@ -1950,7 +1986,6 @@ def __init__( is_separate_draft_engine: bool = False, ) -> None: super().__init__() - del aux_stream_dict # single-stream model; accepted for the factory's call shape if is_separate_draft_engine: raise NotImplementedError( "glm5_next MTP runs one-model speculative decoding only (MTP / MTP_EAGLE_ONE_MODEL)" @@ -1992,7 +2027,13 @@ def __init__( attn_backend=model_config.attn_backend, model_config=model_config, ) - self.mlp = Glm5NextMoE(config, model_config, layer_idx, dtype=dtype) + self.mlp = Glm5NextMoE( + config, + model_config, + layer_idx, + dtype=dtype, + aux_stream=aux_stream_dict.get(AuxStreamType.MoeShared) if aux_stream_dict else None, + ) self.shared_head = Glm5NextMTPHead(config, model_config, dtype) def forward( diff --git a/tensorrt_llm/_torch/moe/fused_moe/fp32_router_gemm.py b/tensorrt_llm/_torch/moe/fused_moe/fp32_router_gemm.py new file mode 100644 index 000000000000..8d581afa7ddc --- /dev/null +++ b/tensorrt_llm/_torch/moe/fused_moe/fp32_router_gemm.py @@ -0,0 +1,57 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Small-batch router GEMM with FP32 weights, accumulation and output.""" + +import torch +import triton +import triton.language as tl + + +@triton.jit +def _router_mv( + X, W, Y, SX: tl.constexpr, E: tl.constexpr, K: tl.constexpr, BN: tl.constexpr, BK: tl.constexpr +): + row = tl.program_id(0) + expert = tl.program_id(1) * BN + tl.arange(0, BN) + channel = tl.arange(0, BK) + x = tl.load(X + row * SX + channel, channel < K, 0).to(tl.float32) + w = tl.load( + W + expert[:, None] * K + channel[None, :], + (expert[:, None] < E) & (channel[None, :] < K), + 0, + ) + out = tl.sum(w * x[None, :], axis=1) + tl.store(Y + row * E + expert, out, expert < E) + + +def fp32_router_gemm(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: + """FP32 router logits for small BF16 batches, preserving FP32 weights. + + x is [tokens, hidden] with contiguous hidden rows; weight is contiguous + [experts, hidden]. The caller selects this path for 1-8 tokens, 288 experts + and hidden size 4096; larger batches use the ordinary GEMM. + """ + rows = x.shape[0] + if rows == 1: + bn, warps = 1, 4 + elif rows == 2: + bn, warps = 1, 8 + elif rows <= 4: + bn, warps = 2, 8 + else: + bn, warps = 2, 4 + assert x.dtype == torch.bfloat16 and weight.dtype == torch.float32 + assert x.stride(-1) == 1 and weight.is_contiguous() + output = torch.empty((x.shape[0], weight.shape[0]), dtype=torch.float32, device=x.device) + _router_mv[(x.shape[0], triton.cdiv(weight.shape[0], bn))]( + x, + weight, + output, + x.stride(0), + weight.shape[0], + weight.shape[1], + bn, + triton.next_power_of_2(weight.shape[1]), + num_warps=warps, + ) + return output diff --git a/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py b/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py index 2a57f034958d..ab1a0f31cd12 100644 --- a/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py +++ b/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py @@ -416,3 +416,57 @@ def test_fp8_cache_storage_helpers_preserve_coalesced_pages(context): torch.testing.assert_close(actual_packed.float().unsqueeze(0), packed) assert torch.count_nonzero(storage.float()[[0, 2, 3, 5, 6, 8]]) == 0 assert torch.count_nonzero(index_pool[:, :, 4:]) == 0 + + +@pytest.mark.skipif( + get_sm_version() not in (100, 103), reason="native sparse MLA requires SM100/103" +) +@pytest.mark.parametrize("rows,width", [(1, 64), (8, 2052)]) +def test_native_sparse_decode_coalesced_rows_and_graph_replay(rows: int, width: int) -> None: + from types import SimpleNamespace + + from tensorrt_llm._torch.attention.backends.sparse.glm_kpool.backend import latent_pool_rows + from tensorrt_llm._torch.attention.backends.sparse.glm_kpool.native_decode import ( + GlmKpoolNativeDecode, + ) + + generator = torch.Generator(device="cuda").manual_seed(6810832) + # Only the middle subpage belongs to this layer. Other subpages contain NaN + # to expose wrong physical addressing instead of accidentally valid values. + storage = torch.full((32, 3, 32, 512), float("nan"), dtype=torch.bfloat16, device="cuda") + latent = storage[:, 1] + latent.copy_( + torch.randn(latent.shape, dtype=latent.dtype, device=latent.device, generator=generator) + ) + kv_rows, base, stride = latent_pool_rows(latent) + q_storage = torch.randn(rows, 32, 512, dtype=torch.bfloat16, device="cuda", generator=generator) + q = q_storage[:, ::2] + indices_storage = torch.empty(rows, width + 7, dtype=torch.int32, device="cuda") + indices = indices_storage[:, :width] + logical = torch.randint(0, 1024, (rows, width), device="cuda", generator=generator) + physical = base + (logical // 32) * stride + logical % 32 + indices.copy_(physical) + indices[:, ::5] = -1 + runner = GlmKpoolNativeDecode() + metadata = SimpleNamespace(effective_workspace=torch.empty(0, dtype=torch.int8, device="cuda")) + runner.prepare_workspace(q, metadata) + scale = 256**-0.5 + runner(q, kv_rows, indices, scale, metadata) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + actual = runner(q, kv_rows, indices, scale, metadata) + for replay in range(3): + q.copy_(torch.randn(q.shape, dtype=q.dtype, device=q.device, generator=generator)) + indices.copy_(physical.roll(replay, dims=1)) + indices[:, replay::5] = -1 + if replay == 1: + indices[-1] = -1 + graph.replay() + valid = indices >= 0 + selected = kv_rows[indices.clamp_min(0).long(), 0].float() + selected.masked_fill_(~valid[:, :, None], 0) + logits = torch.einsum("rhd,rkd->rhk", q.float(), selected) * scale + logits.masked_fill_(~valid[:, None, :], float("-inf")) + probabilities = torch.nan_to_num(logits.softmax(dim=-1)) + expected = torch.einsum("rhk,rkd->rhd", probabilities, selected).to(q.dtype) + torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02) diff --git a/tests/unittest/_torch/moe/test_fp32_router_gemm.py b/tests/unittest/_torch/moe/test_fp32_router_gemm.py new file mode 100644 index 000000000000..d882b2186065 --- /dev/null +++ b/tests/unittest/_torch/moe/test_fp32_router_gemm.py @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +import pytest +import torch + +from tensorrt_llm._torch.moe.fused_moe.fp32_router_gemm import fp32_router_gemm + + +@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") +@pytest.mark.parametrize("rows", [1, 2, 4, 8]) +@pytest.mark.parametrize("strided", [False, True]) +def test_fp32_router_logits_and_selection_replay(rows: int, strided: bool) -> None: + generator = torch.Generator(device="cuda").manual_seed(6810832) + weight = torch.randn(288, 4096, device="cuda", generator=generator) * 0.02 + bias = torch.randn(288, device="cuda", generator=generator) * 0.01 + storage = torch.randn(rows * 2, 4096, dtype=torch.bfloat16, device="cuda", generator=generator) + x = storage[::2] if strided else storage[:rows] + fp32_router_gemm(x, weight) + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + actual = fp32_router_gemm(x, weight) + for _ in range(3): + x.copy_(torch.randn(x.shape, dtype=x.dtype, device=x.device, generator=generator)) + graph.replay() + # Explicit FP32 products avoid any TF32 setting affecting the oracle. + expected = (x.float()[:, None, :] * weight[None, :, :]).sum(dim=-1) + torch.testing.assert_close(actual, expected, atol=1e-5, rtol=1e-5) + actual_ids = torch.topk(actual.sigmoid() + bias, 8, dim=-1).indices + expected_ids = torch.topk(expected.sigmoid() + bias, 8, dim=-1).indices + assert torch.equal(actual_ids, expected_ids) From 5dea8f39479dfcc6f8c785e3e310b25cc8676aa2 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Mon, 21 Sep 2026 00:26:47 -0700 Subject: [PATCH 17/35] [TRTLLM-16018][fix] Fix GLM k-pool offsets and TopK fallback Widen score row indices before stride multiplication so long-context prefill can address score matrices beyond INT32_MAX elements. Use the shared Torch TopK fallback when CuTe DSL is unavailable, avoiding missing native radix scratch buffers. Cover all three score-store branches across the 2^31-element boundary and validate candidate bounds and CUDA graph replay without CuTe DSL. Validation: 66 CPU and 37 GPU tests passed, including rebased KDA parity and BF16 state tests. Signed-off-by: Ruocheng Jia --- .../backends/sparse/glm_kpool/kernels.py | 3 +- .../_torch/models/modeling_glm5_next.py | 4 +- .../sparse/glm_kpool/test_kernels.py | 103 ++++++++++++++++++ .../modeling/test_glm5_next_contracts.py | 22 ++++ 4 files changed, 129 insertions(+), 3 deletions(-) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py index 9b4020b0f895..ca6ce703d728 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/kernels.py @@ -221,7 +221,8 @@ def _kpool_score_kernel( for every row of the program. A group straddling two requests (only at request boundaries) gathers per row instead. """ - r0 = tl.program_id(0) * ROWS + # The FP32 score matrix can exceed 2**31 elements during long-context prefill. + r0 = tl.program_id(0).to(tl.int64) * ROWS j0 = tl.program_id(1) * BP j = j0 + tl.arange(0, BP) d = tl.arange(0, HD) diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next.py b/tensorrt_llm/_torch/models/modeling_glm5_next.py index 03e0c8285968..3f2dd698e284 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next.py @@ -946,7 +946,7 @@ def __init__( torch.zeros(self.index_kpool, self.head_dim, dtype=torch.bfloat16) ) # Reuse DSA's TopK, bounded by each request's candidate count. - # Prefer CuTe DSL radix selection, with CUDA radix as the fallback. + # Torch handles the FP32 fallback without caller-owned CUDA radix scratch. from ..cute_dsl_utils import IS_CUTLASS_DSL_AVAILABLE from ..modules.top_k import TopK, TopKImplementation @@ -956,7 +956,7 @@ def __init__( decode_implementation=( TopKImplementation.CUTE_DSL_RADIX if IS_CUTLASS_DSL_AVAILABLE - else TopKImplementation.CUDA_RADIX + else TopKImplementation.TORCH ), ) diff --git a/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py b/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py index ab1a0f31cd12..3f0458801b7b 100644 --- a/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py +++ b/tests/unittest/_torch/attention/sparse/glm_kpool/test_kernels.py @@ -470,3 +470,106 @@ def test_native_sparse_decode_coalesced_rows_and_graph_replay(rows: int, width: probabilities = torch.nan_to_num(logits.softmax(dim=-1)) expected = torch.einsum("rhk,rkd->rhd", probabilities, selected).to(q.dtype) torch.testing.assert_close(actual, expected, atol=0.02, rtol=0.02) + + +def test_pool_scores_offsets_beyond_int32() -> None: + from tensorrt_llm._torch.attention.backends.sparse.glm_kpool.kernels import _kpool_score_kernel + + # A 1M-token capacity has 2**18 pools. Row 8192 starts at element 2**31. + # Only launch the first pool tile: exercise real pointer arithmetic without + # scoring every pool or touching the entire 8 GiB allocation. + rows, capacity, rows_per_program = 8208, 1 << 18, 16 + required_bytes = rows * capacity * 4 + if torch.cuda.mem_get_info()[0] < required_bytes + (1 << 30): + pytest.skip("large score-offset regression requires 9 GiB of free device memory") + out = torch.empty(rows, capacity, dtype=torch.float32, device="cuda") + q = torch.ones(rows, 1, HD, dtype=torch.bfloat16, device="cuda") + weights = torch.ones(rows, 1, dtype=torch.bfloat16, device="cuda") + pool = torch.ones(2, TPB, 3 * HD, dtype=torch.bfloat16, device="cuda") + pool[1].fill_(2) + tables = torch.tensor([[0], [1]], dtype=torch.int64, device="cuda") + lengths = torch.zeros(rows, dtype=torch.int64, device="cuda") + requests = torch.zeros(rows, dtype=torch.int32, device="cuda") + + for branch in ("invisible", "shared", "request_boundary"): + if branch != "invisible": + lengths.fill_(KPOOL) + if branch == "request_boundary": + requests[-8:] = 1 + _kpool_score_kernel[(rows // rows_per_program, 1)]( + q, + weights, + pool, + tables, + lengths, + requests, + out, + rows, + weights.stride(0), + weights.stride(1), + pool.stride(0), + pool.stride(1), + tables.stride(0), + TPB, + capacity, + 1.0 / HD, + 1.0, + FP32_MIN, + HD=HD, + KPOOL=KPOOL, + H=1, + HP=16, + BP=64, + ROWS=rows_per_program, + PRECISION="tf32", + HAS_REQ=True, + num_warps=4, + ) + expected = torch.full((32, 64), FP32_MIN, device="cuda") + if branch != "invisible": + expected[:, 0] = 1 + if branch == "request_boundary": + expected[-8:, 0] = 2 + torch.testing.assert_close(out[-32:, :64], expected, rtol=0, atol=0) + + +def test_pool_topk_without_cute_dsl_replays_graph(monkeypatch: pytest.MonkeyPatch) -> None: + from types import SimpleNamespace + + from tensorrt_llm._torch import cute_dsl_utils + from tensorrt_llm._torch.models.modeling_glm5_next import Glm5NextIndexer + + monkeypatch.setattr(cute_dsl_utils, "IS_CUTLASS_DSL_AVAILABLE", False) + config = SimpleNamespace( + hidden_size=256, + q_lora_rank=128, + index_n_heads=32, + index_head_dim=128, + index_topk=2048, + index_kpool=4, + index_kpool_always_select_tail=True, + ) + with torch.device("meta"): + indexer = Glm5NextIndexer(config, layer_idx=0) + scores = torch.arange(520, dtype=torch.float32, device="cuda").repeat(3, 1) + lengths = torch.tensor([0, 3, 513], dtype=torch.int32, device="cuda") + selected = torch.empty(3, 512, dtype=torch.int32, device="cuda") + + def select() -> torch.Tensor: + return indexer.pool_top_k( + scores, selected, is_prefill=False, sequence_lengths=lengths, scan_lengths=lengths + ) + + select() + graph = torch.cuda.CUDAGraph() + with torch.cuda.graph(graph): + select() + for bounds in ([0, 3, 513], [520, 1, 0]): + lengths.copy_(torch.tensor(bounds, dtype=torch.int32, device="cuda")) + graph.replay() + for row, count in enumerate(bounds): + actual = selected[row].cpu() + valid = actual[actual >= 0].sort().values + expected = torch.arange(max(0, count - 512), count, dtype=torch.int32) + assert torch.equal(valid, expected) + assert int((actual == -1).sum()) == 512 - min(count, 512) diff --git a/tests/unittest/_torch/modeling/test_glm5_next_contracts.py b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py index 582df8738b2c..e037218e9e38 100644 --- a/tests/unittest/_torch/modeling/test_glm5_next_contracts.py +++ b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py @@ -643,3 +643,25 @@ def test_packed_attention_splits_phases_and_reduces_once(generation_phase, reduc other.assert_not_called() if reduction is not None: reduction.assert_called_once() + + +@pytest.mark.cpu_only +def test_indexer_topk_without_cute_dsl(monkeypatch: pytest.MonkeyPatch) -> None: + from tensorrt_llm._torch import cute_dsl_utils + from tensorrt_llm._torch.models.modeling_glm5_next import Glm5NextIndexer + + monkeypatch.setattr(cute_dsl_utils, "IS_CUTLASS_DSL_AVAILABLE", False) + with torch.device("meta"): + indexer = Glm5NextIndexer(_config().text_config, layer_idx=0) + scores = torch.arange(520, dtype=torch.float32).repeat(3, 1) + lengths = torch.tensor([0, 3, 513], dtype=torch.int32) + selected = torch.empty(3, indexer.select_k, dtype=torch.int32) + # GLM calls this entry for both packed prefill rows and generation rows. + indexer.pool_top_k( + scores, selected, is_prefill=False, sequence_lengths=lengths, scan_lengths=lengths + ) + for row, count in enumerate(lengths.tolist()): + valid = selected[row][selected[row] >= 0].sort().values + expected = torch.arange(max(0, count - indexer.select_k), count, dtype=torch.int32) + assert torch.equal(valid, expected) + assert int((selected[row] == -1).sum()) == indexer.select_k - min(count, indexer.select_k) From 901fc65bc49b7efad357718a5dcfe5ee57205889 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Tue, 22 Sep 2026 01:02:46 -0700 Subject: [PATCH 18/35] [None][refactor] Reuse FP32 linear for GLM router Remove the GLM-specific small-batch router kernel and its dedicated tests. Use F.linear with FP32 inputs and weights for router logits. Signed-off-by: Ruocheng Jia --- .../_torch/models/modeling_glm5_next.py | 11 ---- .../_torch/moe/fused_moe/fp32_router_gemm.py | 57 ------------------- .../_torch/moe/test_fp32_router_gemm.py | 30 ---------- 3 files changed, 98 deletions(-) delete mode 100644 tensorrt_llm/_torch/moe/fused_moe/fp32_router_gemm.py delete mode 100644 tests/unittest/_torch/moe/test_fp32_router_gemm.py diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next.py b/tensorrt_llm/_torch/models/modeling_glm5_next.py index 3f2dd698e284..a101469f5f2c 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next.py @@ -55,7 +55,6 @@ from ..modules.linear import Linear, TensorParallelMode from ..modules.multi_stream_utils import maybe_execute_in_parallel from ..modules.rms_norm import RMSNorm -from ..moe.fused_moe.fp32_router_gemm import fp32_router_gemm from ..pyexecutor.config_utils import unwrap_glm5_next_text_config from ..utils import AuxStreamType from .checkpoints.hf.glm5_next_weight_mapper import ( @@ -1449,16 +1448,6 @@ def __init__(self, config: PretrainedConfig, moe_backend: str = "CUTLASS") -> No def forward(self, hidden_states: torch.Tensor) -> torch.Tensor: flat = hidden_states.reshape(-1, self.hidden_size) - if ( - flat.is_cuda - and flat.dtype == torch.bfloat16 - and self.weight.dtype == torch.float32 - and self.weight.shape == (288, 4096) - and self.weight.is_contiguous() - and flat.stride(-1) == 1 - and 1 <= flat.shape[0] <= 8 - ): - return fp32_router_gemm(flat, self.weight) return torch.nn.functional.linear(flat.float(), self.weight) diff --git a/tensorrt_llm/_torch/moe/fused_moe/fp32_router_gemm.py b/tensorrt_llm/_torch/moe/fused_moe/fp32_router_gemm.py deleted file mode 100644 index 8d581afa7ddc..000000000000 --- a/tensorrt_llm/_torch/moe/fused_moe/fp32_router_gemm.py +++ /dev/null @@ -1,57 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Small-batch router GEMM with FP32 weights, accumulation and output.""" - -import torch -import triton -import triton.language as tl - - -@triton.jit -def _router_mv( - X, W, Y, SX: tl.constexpr, E: tl.constexpr, K: tl.constexpr, BN: tl.constexpr, BK: tl.constexpr -): - row = tl.program_id(0) - expert = tl.program_id(1) * BN + tl.arange(0, BN) - channel = tl.arange(0, BK) - x = tl.load(X + row * SX + channel, channel < K, 0).to(tl.float32) - w = tl.load( - W + expert[:, None] * K + channel[None, :], - (expert[:, None] < E) & (channel[None, :] < K), - 0, - ) - out = tl.sum(w * x[None, :], axis=1) - tl.store(Y + row * E + expert, out, expert < E) - - -def fp32_router_gemm(x: torch.Tensor, weight: torch.Tensor) -> torch.Tensor: - """FP32 router logits for small BF16 batches, preserving FP32 weights. - - x is [tokens, hidden] with contiguous hidden rows; weight is contiguous - [experts, hidden]. The caller selects this path for 1-8 tokens, 288 experts - and hidden size 4096; larger batches use the ordinary GEMM. - """ - rows = x.shape[0] - if rows == 1: - bn, warps = 1, 4 - elif rows == 2: - bn, warps = 1, 8 - elif rows <= 4: - bn, warps = 2, 8 - else: - bn, warps = 2, 4 - assert x.dtype == torch.bfloat16 and weight.dtype == torch.float32 - assert x.stride(-1) == 1 and weight.is_contiguous() - output = torch.empty((x.shape[0], weight.shape[0]), dtype=torch.float32, device=x.device) - _router_mv[(x.shape[0], triton.cdiv(weight.shape[0], bn))]( - x, - weight, - output, - x.stride(0), - weight.shape[0], - weight.shape[1], - bn, - triton.next_power_of_2(weight.shape[1]), - num_warps=warps, - ) - return output diff --git a/tests/unittest/_torch/moe/test_fp32_router_gemm.py b/tests/unittest/_torch/moe/test_fp32_router_gemm.py deleted file mode 100644 index d882b2186065..000000000000 --- a/tests/unittest/_torch/moe/test_fp32_router_gemm.py +++ /dev/null @@ -1,30 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -import pytest -import torch - -from tensorrt_llm._torch.moe.fused_moe.fp32_router_gemm import fp32_router_gemm - - -@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA") -@pytest.mark.parametrize("rows", [1, 2, 4, 8]) -@pytest.mark.parametrize("strided", [False, True]) -def test_fp32_router_logits_and_selection_replay(rows: int, strided: bool) -> None: - generator = torch.Generator(device="cuda").manual_seed(6810832) - weight = torch.randn(288, 4096, device="cuda", generator=generator) * 0.02 - bias = torch.randn(288, device="cuda", generator=generator) * 0.01 - storage = torch.randn(rows * 2, 4096, dtype=torch.bfloat16, device="cuda", generator=generator) - x = storage[::2] if strided else storage[:rows] - fp32_router_gemm(x, weight) - graph = torch.cuda.CUDAGraph() - with torch.cuda.graph(graph): - actual = fp32_router_gemm(x, weight) - for _ in range(3): - x.copy_(torch.randn(x.shape, dtype=x.dtype, device=x.device, generator=generator)) - graph.replay() - # Explicit FP32 products avoid any TF32 setting affecting the oracle. - expected = (x.float()[:, None, :] * weight[None, :, :]).sum(dim=-1) - torch.testing.assert_close(actual, expected, atol=1e-5, rtol=1e-5) - actual_ids = torch.topk(actual.sigmoid() + bias, 8, dim=-1).indices - expected_ids = torch.topk(expected.sigmoid() + bias, 8, dim=-1).indices - assert torch.equal(actual_ids, expected_ids) From 2a024282cfac52370c5903accaddbabdfa9d3ac7 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Tue, 22 Sep 2026 02:15:21 -0700 Subject: [PATCH 19/35] [None][refactor] Align GLM sparse attention with shared interfaces Pass MLA geometry through the shared attention factory, including NoPE support. Move GLM recurrent metadata into its own module, reuse prefer_pinned directly, and guard the module-level FlashMLA import. Expose raw cache slot indices through KVCacheManagerV2 and replace the GLM private-method call. Cover the shared MLA factory path and extend the existing cache-index test. Signed-off-by: Ruocheng Jia --- .../backends/sparse/glm_kpool/__init__.py | 3 +- .../backends/sparse/glm_kpool/backend.py | 40 ++----- .../sparse/glm_kpool/cache_manager.py | 107 ++---------------- .../backends/sparse/glm_kpool/metadata.py | 86 ++++++++++++++ .../backends/sparse/glm_kpool/params.py | 11 -- .../_torch/attention/backends/utils.py | 2 +- .../_torch/models/modeling_glm5_next.py | 11 +- .../kv_cache/kv_cache_manager_v2.py | 14 +++ .../sparse/glm_kpool/test_glm_kpool.py | 40 +++++++ .../executor/test_per_layer_head_dim.py | 4 + 10 files changed, 175 insertions(+), 143 deletions(-) create mode 100644 tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/metadata.py diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/__init__.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/__init__.py index 21ae44a85268..c8d4638792ee 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/__init__.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/__init__.py @@ -15,7 +15,8 @@ """glm_kpool: the GLM-5.3-Flash pool-compressed sparse-MLA backend.""" from .backend import INDEX_SENTINEL, GlmKpoolSparseAttention, latent_pool_rows, paged_slot_indices -from .cache_manager import Glm5NextCacheManager, Glm5NextMamba2Metadata +from .cache_manager import Glm5NextCacheManager +from .metadata import Glm5NextMamba2Metadata from .params import GlmKpoolBackendForwardArgs, GlmKpoolSparseParams __all__ = [ diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py index b947b7438e8d..74e39a86b3cb 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py @@ -27,7 +27,6 @@ from __future__ import annotations -from collections.abc import Callable from dataclasses import dataclass import torch @@ -46,17 +45,10 @@ from .native_decode import GlmKpoolNativeDecode from .params import INDEX_SENTINEL, GlmKpoolSparseParams - -def _flash_mla_sparse_fwd() -> Callable[..., tuple[torch.Tensor, torch.Tensor, torch.Tensor]]: - """Resolve the FlashMLA sparse kernel lazily.""" - try: - from tensorrt_llm.flash_mla import flash_mla_sparse_fwd - except ImportError as exc: # pragma: no cover - wheel always bundles it - raise RuntimeError( - "glm_kpool sparse MLA requires tensorrt_llm.flash_mla." - "flash_mla_sparse_fwd, which this build does not provide" - ) from exc - return flash_mla_sparse_fwd +try: + from tensorrt_llm.flash_mla import flash_mla_sparse_fwd +except ImportError: + flash_mla_sparse_fwd = None def paged_slot_indices( @@ -166,28 +158,18 @@ def __init__( raise TypeError( f"GlmKpoolSparseAttention needs GlmKpoolSparseParams, got {type(sparse_params)}" ) - if head_dim != sparse_params.kv_lora_rank: + if mla_params is None: + raise ValueError("glm_kpool requires MLA parameters from create_attention") + if head_dim != mla_params.kv_lora_rank: raise ValueError( "glm_kpool consumes absorbed latent-space queries: head_dim " - f"({head_dim}) must equal kv_lora_rank ({sparse_params.kv_lora_rank})" + f"({head_dim}) must equal kv_lora_rank ({mla_params.kv_lora_rank})" ) if pos_embd_params is not None: raise ValueError( "glm_kpool is fully NoPE; positional embedding parameters have no " "meaning on this branch" ) - if mla_params is None: - # The standard create_attention MLA path asserts qk_rope_head_dim>0 - # (the rope'd DeepSeek geometry), so this fully-NoPE branch states - # its MLA identity itself instead of loosening the shared assert. - mla_params = MLAParams( - q_lora_rank=sparse_params.q_lora_rank, - kv_lora_rank=sparse_params.kv_lora_rank, - qk_rope_head_dim=0, - qk_nope_head_dim=sparse_params.qk_nope_head_dim, - v_head_dim=sparse_params.v_head_dim, - rope_append=False, - ) if mla_params.qk_rope_head_dim != 0: raise ValueError( f"glm_kpool is fully NoPE; got qk_rope_head_dim={mla_params.qk_rope_head_dim}" @@ -214,7 +196,7 @@ def __init__( #: Softmax scale of the *unabsorbed* q . k product over #: ``qk_nope_head_dim``; absorption reassociates the matmuls but the #: score scale is unchanged. - self.softmax_scale = float(sparse_params.qk_nope_head_dim) ** -0.5 + self.softmax_scale = float(self.qk_nope_head_dim) ** -0.5 self._native_decode = GlmKpoolNativeDecode() @classmethod @@ -515,7 +497,9 @@ def _dispatch_sparse_core( q_padded = q_latent.new_zeros((q_latent.shape[0], kernel_heads, q_latent.shape[2])) q_padded[:, :local_heads, :] = q_latent q_latent = q_padded - out, _, _ = _flash_mla_sparse_fwd()( + if flash_mla_sparse_fwd is None: + raise RuntimeError("glm_kpool sparse attention requires FlashMLA in this build") + out, _, _ = flash_mla_sparse_fwd( q_latent, kv_rows, topk_rows.unsqueeze(1), diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py index 189f006d8634..f2e5d4bb4df7 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py @@ -12,12 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""GLM hybrid cache manager and persistent sparse-attention metadata. - -Kimi KDA metadata handles recurrent state and replay scheduling. GLM adds raw -slot tables with fixed device addresses plus host prefill schedules. Visible -lengths come from TRTLLM kv_lens_cuda, including overlap and MTP corrections. -""" +"""GLM hybrid cache manager for recurrent, latent KV, and indexer state.""" from __future__ import annotations @@ -25,79 +20,11 @@ import torch -from tensorrt_llm._torch.modules.kimi_kda.kimi_k3_mamba_metadata import KimiK3MambaMetadata from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import Role from tensorrt_llm._torch.pyexecutor.kv_cache.mamba_cache_manager import MambaHybridCacheManagerV2 from tensorrt_llm.runtime.kv_cache_manager_v2 import BufferConfig -class Glm5NextMamba2Metadata(KimiK3MambaMetadata): - def __init__(self, max_batch_size: int, chunk_size: int, max_num_tokens: int) -> None: - super().__init__(max_batch_size, chunk_size, max_num_tokens) - from tensorrt_llm._utils import prefer_pinned - - self._glm_pin = prefer_pinned() - self.glm_block_tables: torch.Tensor | None = None - self._glm_block_tables_cpu: torch.Tensor | None = None - self.glm_cached_lens_host: list[int] = [] - self.glm_ctx_cu_seqlens: list[int] = [0] - - def _glm_ensure_tables(self, width: int) -> None: - width = max(1, int(width)) - if self.glm_block_tables is None: - self.glm_block_tables = torch.zeros( - self.max_batch_size, width, dtype=torch.long, device="cuda" - ) - self._glm_block_tables_cpu = torch.zeros( - self.max_batch_size, width, dtype=torch.long, pin_memory=self._glm_pin - ) - elif self.glm_block_tables.shape[1] < width: - raise RuntimeError( - "glm5_next block-table buffer would need to grow from " - f"{self.glm_block_tables.shape[1]} to {width} pages mid-run; " - "captured CUDA graphs would keep reading the old buffer" - ) - - def prepare(self, attn_metadata) -> None: - super().prepare(attn_metadata) - manager = attn_metadata.kv_cache_manager - kv_params = attn_metadata.kv_cache_params - request_ids = attn_metadata.request_ids - if ( - manager is None - or not hasattr(manager, "get_batch_slot_tables") - or kv_params is None - or kv_params.num_cached_tokens_per_seq is None - or request_ids is None - ): - return - - batch = attn_metadata.seq_lens.shape[0] - num_contexts = int(attn_metadata.num_contexts) - lens = [int(x) for x in attn_metadata.seq_lens[:batch]] - cached_src = kv_params.num_cached_tokens_per_seq - if isinstance(cached_src, torch.Tensor): - cached = [int(x) for x in cached_src[:batch]] - else: - cached = [int(cached_src[i]) for i in range(batch)] - - self.glm_cached_lens_host = cached - cu = [0] - for length in lens[:num_contexts]: - cu.append(cu[-1] + length) - self.glm_ctx_cu_seqlens = cu - - width = int(getattr(manager, "max_blocks_per_seq", 0)) or 1 - self._glm_ensure_tables(width) - pages = manager.get_batch_slot_tables(list(request_ids)[:batch]) - staging = self._glm_block_tables_cpu - staging[:batch].zero_() - for row, page_ids in enumerate(pages): - if page_ids: - staging[row, : len(page_ids)].copy_(torch.as_tensor(page_ids, dtype=torch.long)) - self.glm_block_tables[:batch].copy_(staging[:batch], non_blocking=True) - - class Glm5NextCacheManager(MambaHybridCacheManagerV2): """Manage KDA state, latent KV and indexer buffers in one V2 lifecycle. @@ -141,37 +68,21 @@ def get_index_state_buffer(self, layer_idx: int) -> torch.Tensor | None: kv_layout="NHD", ) - def _sparse_pool_id(self) -> int: - """Require one V2 layer group for the shared sparse-layer block table. + def get_batch_slot_tables(self, request_ids: Sequence[int]) -> list[list[int]]: + """Return raw slot IDs shared by the sparse layers' latent and indexer views. - KEY and INDEX_KEY views must use the same raw slot space across sparse layers. + PP ranks without local sparse layers return empty rows. """ - pools = { - self.layer_to_pool_mapping_dict[self.layer_offsets[layer_id]] - for layer_id in self.sparse_layer_ids - if layer_id in self.layer_offsets - } + local_layers = [i for i in self.sparse_layer_ids if i in self.layer_offsets] + if not local_layers: + return [[] for _ in request_ids] + pools = {self.layer_to_pool_mapping_dict[self.layer_offsets[i]] for i in local_layers} if len(pools) != 1: raise ValueError( f"glm5_next sparse layers span V2 layer groups {sorted(pools)}; " "the slot-indexed latent/index views require a single group" ) - return pools.pop() - - def get_batch_slot_tables(self, request_ids: Sequence[int]) -> list[list[int]]: - """Return raw base-slot IDs, without V2's per-layer page-index scaling. - - Both latent and indexer views fold that scaling into their slot stride. - PP ranks without local sparse layers return empty rows. - """ - if not any(layer_id in self.layer_offsets for layer_id in self.sparse_layer_ids): - return [[] for _ in request_ids] - return self._get_batch_cache_indices_by_pool_id( - list(request_ids), - pool_id=self._sparse_pool_id(), - is_kv_aggregate=False, - index_scale=1, - ) + return self.get_batch_base_page_indices(list(request_ids), layer_idx=local_layers[0]) def get_latent_state_buffer(self, layer_idx: int) -> torch.Tensor | None: """Return a slot-major [slots, tokens_per_block, num_kv_heads, head_dim] view. diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/metadata.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/metadata.py new file mode 100644 index 000000000000..279515981d55 --- /dev/null +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/metadata.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Persistent GLM slot tables and host prefill schedules alongside KDA metadata.""" + +from __future__ import annotations + +import torch + +from tensorrt_llm._torch.modules.kimi_kda.kimi_k3_mamba_metadata import KimiK3MambaMetadata +from tensorrt_llm._utils import prefer_pinned + + +class Glm5NextMamba2Metadata(KimiK3MambaMetadata): + def __init__(self, max_batch_size: int, chunk_size: int, max_num_tokens: int) -> None: + super().__init__(max_batch_size, chunk_size, max_num_tokens) + self.glm_block_tables: torch.Tensor | None = None + self._glm_block_tables_cpu: torch.Tensor | None = None + self.glm_cached_lens_host: list[int] = [] + self.glm_ctx_cu_seqlens: list[int] = [0] + + def _glm_ensure_tables(self, width: int) -> None: + width = max(1, int(width)) + if self.glm_block_tables is None: + self.glm_block_tables = torch.zeros( + self.max_batch_size, width, dtype=torch.long, device="cuda" + ) + self._glm_block_tables_cpu = torch.zeros( + self.max_batch_size, width, dtype=torch.long, pin_memory=prefer_pinned() + ) + elif self.glm_block_tables.shape[1] < width: + raise RuntimeError( + "glm5_next block-table buffer would need to grow from " + f"{self.glm_block_tables.shape[1]} to {width} pages mid-run; " + "captured CUDA graphs would keep reading the old buffer" + ) + + def prepare(self, attn_metadata) -> None: + super().prepare(attn_metadata) + manager = attn_metadata.kv_cache_manager + kv_params = attn_metadata.kv_cache_params + request_ids = attn_metadata.request_ids + if ( + manager is None + or not hasattr(manager, "get_batch_slot_tables") + or kv_params is None + or kv_params.num_cached_tokens_per_seq is None + or request_ids is None + ): + return + + batch = attn_metadata.seq_lens.shape[0] + num_contexts = int(attn_metadata.num_contexts) + lens = [int(x) for x in attn_metadata.seq_lens[:batch]] + cached_src = kv_params.num_cached_tokens_per_seq + if isinstance(cached_src, torch.Tensor): + cached = [int(x) for x in cached_src[:batch]] + else: + cached = [int(cached_src[i]) for i in range(batch)] + + self.glm_cached_lens_host = cached + cu = [0] + for length in lens[:num_contexts]: + cu.append(cu[-1] + length) + self.glm_ctx_cu_seqlens = cu + + width = int(getattr(manager, "max_blocks_per_seq", 0)) or 1 + self._glm_ensure_tables(width) + pages = manager.get_batch_slot_tables(list(request_ids)[:batch]) + staging = self._glm_block_tables_cpu + staging[:batch].zero_() + for row, page_ids in enumerate(pages): + if page_ids: + staging[row, : len(page_ids)].copy_(torch.as_tensor(page_ids, dtype=torch.long)) + self.glm_block_tables[:batch].copy_(staging[:batch], non_blocking=True) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/params.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/params.py index 43988dd43025..17a3d7265f07 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/params.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/params.py @@ -35,17 +35,6 @@ class GlmKpoolSparseParams(SparseParams): """Lowered runtime parameters for the GLM k-pool sparse-MLA backend.""" algorithm: Literal["glm_kpool"] = field(init=False, default="glm_kpool") - #: Latent (compressed KV) width; also the absorbed query head width and - #: the kernel's d_qk == d_v. 512 on this checkpoint. - kv_lora_rank: int = 512 - #: Pre-absorption query/key head width; sets the softmax scale. Fully - #: NoPE: there is no rope component on top of it. - qk_nope_head_dim: int = 256 - #: Low-rank query bottleneck width; carried into ``MLAParams`` so the - #: backend's MLA identity states the real checkpoint geometry. - q_lora_rank: int = 1536 - #: Per-head value width after the absorbed V projection. - v_head_dim: int = 256 #: Number of key positions the expanded selection may cover. index_topk: int = 2048 #: Members per compressed pool. diff --git a/tensorrt_llm/_torch/attention/backends/utils.py b/tensorrt_llm/_torch/attention/backends/utils.py index 79c0876f6679..ec33ca4fa0c3 100644 --- a/tensorrt_llm/_torch/attention/backends/utils.py +++ b/tensorrt_llm/_torch/attention/backends/utils.py @@ -91,7 +91,7 @@ def create_attention( if is_mla_enable: assert attn_cls.support_mla( ), f"MLA is not supported for {backend_name} backend" - assert (q_lora_rank > 0 and kv_lora_rank > 0 and qk_rope_head_dim > 0 + assert (q_lora_rank > 0 and kv_lora_rank > 0 and qk_rope_head_dim >= 0 and qk_nope_head_dim > 0 and v_head_dim > 0) mla_params = MLAParams( q_lora_rank=q_lora_rank, diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next.py b/tensorrt_llm/_torch/models/modeling_glm5_next.py index a101469f5f2c..c46e83c84a62 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next.py @@ -1087,11 +1087,14 @@ def __init__( head_dim=self.kv_lora_rank, num_kv_heads=1, dtype=dtype, + is_mla_enable=True, + q_lora_rank=self.q_lora_rank, + kv_lora_rank=self.kv_lora_rank, + qk_rope_head_dim=0, + qk_nope_head_dim=self.qk_nope_head_dim, + v_head_dim=self.v_head_dim, + rope_append=False, sparse_params=GlmKpoolSparseParams( - kv_lora_rank=self.kv_lora_rank, - qk_nope_head_dim=self.qk_nope_head_dim, - q_lora_rank=self.q_lora_rank, - v_head_dim=self.v_head_dim, index_topk=self.indexer.index_topk, index_kpool=self.indexer.index_kpool, index_always_select_tail=self.indexer.always_select_tail, diff --git a/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py b/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py index bb08cc2cd3e9..1b38f4047d8a 100644 --- a/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py +++ b/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py @@ -5157,6 +5157,20 @@ def get_batch_cache_indices( index_scale=index_scale, ) + def get_batch_base_page_indices( + self, request_ids: List[int], layer_idx: int + ) -> List[List[int]]: + """Return owned raw slot IDs for a layer's pool, preserving invalid entries. + + Unlike get_batch_cache_indices, these indices do not include the layer's + page-index scale or KV aggregation. Slot-major auxiliary cache views use + them directly. The max_blocks_per_seq padding is excluded. + """ + pool_id = self.layer_to_pool_mapping_dict[self.layer_offsets[layer_idx]] + return self._get_batch_cache_indices_by_pool_id( + request_ids, pool_id=pool_id, is_kv_aggregate=False, index_scale=1 + ) + def _get_batch_cache_indices_by_pool_id( self, request_ids: List[int], diff --git a/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py b/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py index 505d186d042c..d008fbbe698d 100644 --- a/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py +++ b/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py @@ -37,6 +37,46 @@ pytestmark = pytest.mark.cpu_only +def test_nope_mla_geometry_uses_shared_factory(monkeypatch): + from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttention + from tensorrt_llm._torch.attention.backends.utils import create_attention + + def initialize_base(self, layer_idx, num_heads, head_dim, num_kv_heads, **kwargs): + self.mla_params = kwargs["mla_params"] + self.num_kv_heads = num_kv_heads + for name in ("q_lora_rank", "kv_lora_rank", "qk_nope_head_dim", "v_head_dim"): + setattr(self, name, getattr(self.mla_params, name)) + + monkeypatch.setattr(TrtllmAttention, "__init__", initialize_base) + kwargs = dict( + backend_name="TRTLLM", + layer_idx=0, + num_heads=16, + head_dim=512, + num_kv_heads=1, + is_mla_enable=True, + q_lora_rank=1536, + kv_lora_rank=512, + qk_rope_head_dim=0, + qk_nope_head_dim=256, + v_head_dim=256, + rope_append=False, + sparse_params=GlmKpoolSparseParams(), + ) + backend = create_attention(**kwargs) + assert isinstance(backend, GlmKpoolSparseAttention) + assert backend.mla_params.qk_rope_head_dim == 0 + assert backend.mla_params.rope_append is False + assert (backend.q_lora_rank, backend.kv_lora_rank, backend.v_head_dim) == (1536, 512, 256) + assert backend.softmax_scale == 256**-0.5 + with pytest.raises(ValueError, match="must equal kv_lora_rank"): + create_attention(**{**kwargs, "head_dim": 256}) + with pytest.raises(ValueError, match="fully NoPE"): + create_attention(**{**kwargs, "qk_rope_head_dim": 64}) + with pytest.raises(AssertionError): + create_attention(**{**kwargs, "qk_rope_head_dim": -1}) + + @pytest.fixture def forward_case(): backend = object.__new__(GlmKpoolSparseAttention) diff --git a/tests/unittest/_torch/executor/test_per_layer_head_dim.py b/tests/unittest/_torch/executor/test_per_layer_head_dim.py index 38b1e0a12ef3..f8ba2c452fbb 100644 --- a/tests/unittest/_torch/executor/test_per_layer_head_dim.py +++ b/tests/unittest/_torch/executor/test_per_layer_head_dim.py @@ -95,6 +95,10 @@ def test_batch_cache_indices_skip_padded_entries(self): for base_page_index in base_page_indices[: kv_cache.num_blocks] ] self.assertEqual(result, [expected]) + raw = list(base_page_indices[: kv_cache.num_blocks]) + for layer_idx in range(mgr.num_local_layers): + self.assertEqual(mgr.get_batch_base_page_indices([7], layer_idx), [raw]) + self.assertEqual(mgr.get_batch_cache_indices([7], layer_idx), [expected]) finally: mgr.shutdown() From 76d5471ee3f8f2531a05727e36ef54c11e1f97a7 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Tue, 22 Sep 2026 04:20:07 -0700 Subject: [PATCH 20/35] [None][refactor] Query GLM raw cache slots through the public layer API Add a keyword-only raw_indices option to the existing V2 cache query, preserving default index conversion for other callers. GLM checks that its sparse layers share one pool and requests raw slots by layer ID. Remove the separate raw-slot accessor and extend the existing index tests to cover both modes and block-count limits. Signed-off-by: Ruocheng Jia --- .../sparse/glm_kpool/cache_manager.py | 4 ++- .../kv_cache/kv_cache_manager_v2.py | 27 +++++++------------ .../executor/test_per_layer_head_dim.py | 15 +++++++---- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py index f2e5d4bb4df7..e4aa9d419147 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py @@ -82,7 +82,9 @@ def get_batch_slot_tables(self, request_ids: Sequence[int]) -> list[list[int]]: f"glm5_next sparse layers span V2 layer groups {sorted(pools)}; " "the slot-indexed latent/index views require a single group" ) - return self.get_batch_base_page_indices(list(request_ids), layer_idx=local_layers[0]) + return self.get_batch_cache_indices( + list(request_ids), layer_idx=local_layers[0], raw_indices=True + ) def get_latent_state_buffer(self, layer_idx: int) -> torch.Tensor | None: """Return a slot-major [slots, tokens_per_block, num_kv_heads, head_dim] view. diff --git a/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py b/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py index 1b38f4047d8a..4205de376ff5 100644 --- a/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py +++ b/tensorrt_llm/_torch/pyexecutor/kv_cache/kv_cache_manager_v2.py @@ -5142,35 +5142,28 @@ def get_batch_cache_indices( request_ids: List[int], layer_idx: Optional[int] = None, num_blocks_per_seq: Optional[Sequence[int]] = None, + *, + raw_indices: bool = False, ) -> List[List[int]]: + """Return cache indices for a layer, or pool 0 when no layer is given. + + Set raw_indices for slot-major views that need base slot IDs without + page-index scaling or KV aggregation. + """ if layer_idx is None: pool_id = 0 - index_scale = None + index_scale = 1 if raw_indices else None else: pool_id = self.layer_to_pool_mapping_dict[self.layer_offsets[layer_idx]] - index_scale = self.get_layer_page_index_scale(layer_idx) + index_scale = 1 if raw_indices else self.get_layer_page_index_scale(layer_idx) return self._get_batch_cache_indices_by_pool_id( request_ids, pool_id=pool_id, - is_kv_aggregate=True, + is_kv_aggregate=not raw_indices, num_blocks_per_seq=num_blocks_per_seq, index_scale=index_scale, ) - def get_batch_base_page_indices( - self, request_ids: List[int], layer_idx: int - ) -> List[List[int]]: - """Return owned raw slot IDs for a layer's pool, preserving invalid entries. - - Unlike get_batch_cache_indices, these indices do not include the layer's - page-index scale or KV aggregation. Slot-major auxiliary cache views use - them directly. The max_blocks_per_seq padding is excluded. - """ - pool_id = self.layer_to_pool_mapping_dict[self.layer_offsets[layer_idx]] - return self._get_batch_cache_indices_by_pool_id( - request_ids, pool_id=pool_id, is_kv_aggregate=False, index_scale=1 - ) - def _get_batch_cache_indices_by_pool_id( self, request_ids: List[int], diff --git a/tests/unittest/_torch/executor/test_per_layer_head_dim.py b/tests/unittest/_torch/executor/test_per_layer_head_dim.py index f8ba2c452fbb..efe26d95a39b 100644 --- a/tests/unittest/_torch/executor/test_per_layer_head_dim.py +++ b/tests/unittest/_torch/executor/test_per_layer_head_dim.py @@ -97,7 +97,9 @@ def test_batch_cache_indices_skip_padded_entries(self): self.assertEqual(result, [expected]) raw = list(base_page_indices[: kv_cache.num_blocks]) for layer_idx in range(mgr.num_local_layers): - self.assertEqual(mgr.get_batch_base_page_indices([7], layer_idx), [raw]) + self.assertEqual( + mgr.get_batch_cache_indices([7], layer_idx, raw_indices=True), [raw] + ) self.assertEqual(mgr.get_batch_cache_indices([7], layer_idx), [expected]) finally: mgr.shutdown() @@ -112,11 +114,14 @@ def test_batch_cache_indices_honor_requested_blocks(self): try: mgr.add_dummy_requests([7], [16]) - all_indices = mgr.get_batch_cache_indices([7]) - requested_indices = mgr.get_batch_cache_indices([7], num_blocks_per_seq=[1]) + for raw_indices in (False, True): + all_indices = mgr.get_batch_cache_indices([7], raw_indices=raw_indices) + requested_indices = mgr.get_batch_cache_indices( + [7], num_blocks_per_seq=[1], raw_indices=raw_indices + ) - self.assertGreater(len(all_indices[0]), 1) - self.assertEqual(requested_indices, [all_indices[0][:1]]) + self.assertGreater(len(all_indices[0]), 1) + self.assertEqual(requested_indices, [all_indices[0][:1]]) finally: mgr.shutdown() From 58920297a027570499a12d7523e439c65c09c740 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Tue, 22 Sep 2026 05:35:26 -0700 Subject: [PATCH 21/35] [None][fix] Preserve GLM 64-head dispatch in the updated KDA backend Instantiate the optimized KDA schedules for 64 heads so GLM decode remains supported when the new main dispatcher selects a non-compact schedule. Signed-off-by: Ruocheng Jia --- cpp/tensorrt_llm/kernels/kdaDecode/kdaDecodeOptimized.cu | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/tensorrt_llm/kernels/kdaDecode/kdaDecodeOptimized.cu b/cpp/tensorrt_llm/kernels/kdaDecode/kdaDecodeOptimized.cu index dec781febc1d..7ac3467437fb 100644 --- a/cpp/tensorrt_llm/kernels/kdaDecode/kdaDecodeOptimized.cu +++ b/cpp/tensorrt_llm/kernels/kdaDecode/kdaDecodeOptimized.cu @@ -740,6 +740,7 @@ void dispatchKdaDecodeOptimizedHeads(KdaDecodeParams const& params, cudaStream_t case 24: TLLM_CUDA_CHECK((launchKernelSchedule(params, stream))); break; case 32: TLLM_CUDA_CHECK((launchKernelSchedule(params, stream))); break; case 48: TLLM_CUDA_CHECK((launchKernelSchedule(params, stream))); break; + case 64: TLLM_CUDA_CHECK((launchKernelSchedule(params, stream))); break; case 96: TLLM_CUDA_CHECK((launchKernelSchedule(params, stream))); break; default: TLLM_CHECK_WITH_INFO(false, "Optimized KDA decode does not support numHeads=%d", params.numHeads); } From eaf9e324db070ee364b83361ad323a7366ea9ade Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Tue, 22 Sep 2026 10:13:28 -0700 Subject: [PATCH 22/35] [None][fix] Declare GLM transceiver preference and simplify attribute access Signed-off-by: Ruocheng Jia --- .../backends/sparse/glm_kpool/backend.py | 19 ++++++---- .../backends/sparse/glm_kpool/metadata.py | 2 +- .../checkpoints/hf/glm5_next_weight_mapper.py | 2 +- .../_torch/models/modeling_glm5_next.py | 37 +++++++++---------- .../models/modeling_glm5_next_vision.py | 12 ++++-- tests/unittest/llmapi/test_llm_args.py | 14 ++++++- 6 files changed, 51 insertions(+), 35 deletions(-) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py index 74e39a86b3cb..16eee8b539e7 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py @@ -223,13 +223,13 @@ def _cache_state(self, metadata) -> _GlmKpoolCacheState: "metadata (TrtllmAttentionMetadata); got None. The backend derives " "its cache pools, block tables, and visible lengths from it." ) - manager = getattr(metadata, "kv_cache_manager", None) + manager = metadata.kv_cache_manager if manager is None: raise ValueError( "glm_kpool metadata has no kv_cache_manager; the hybrid " "KVCacheManagerV2 owns the latent/indexer pools" ) - mamba_metadata = getattr(metadata, "mamba_metadata", None) + mamba_metadata = metadata.mamba_metadata if mamba_metadata is None or mamba_metadata is False: raise ValueError( "glm_kpool requires prepared metadata: call metadata.prepare() " @@ -254,7 +254,7 @@ def _cache_state(self, metadata) -> _GlmKpoolCacheState: "glm_kpool requires prepared glm_block_tables; call metadata.prepare() " "with Glm5NextMamba2Metadata before eager execution or CUDA graph capture" ) - kv_lens = getattr(metadata, "kv_lens_cuda", None) + kv_lens = metadata.kv_lens_cuda if kv_lens is None: raise ValueError("glm_kpool requires prepared metadata.kv_lens_cuda") return _GlmKpoolCacheState( @@ -602,21 +602,24 @@ def forward( raise ValueError(f"glm_kpool forward is phase-explicit, got {input_type!r}") state = self._cache_state(metadata) kv_rows, _, _ = latent_pool_rows(state.latent_pool) + num_query_tokens, num_query_heads, head_dim = q.shape + num_cache_rows = kv_rows.shape[0] native_supported = ( q.is_cuda and q.dtype == kv_rows.dtype == torch.bfloat16 - and q.shape[1:] == (16, 512) - and kv_rows.shape[0] % 32 == 0 + and (num_query_heads, head_dim) == (16, 512) + and num_cache_rows % 32 == 0 and get_sm_version() in (100, 103) ) if native_supported: - # Reserve during prefill/profiling too, before sizing the KV pool. - # Scratch is shared through metadata; each backend owns only counters. + # Reserve shared scratch and per-layer counters during prefill/profiling, + # even though only small generation batches use the native kernel. + # Deferring this allocation to decode would miss it in KV-cache sizing. self._native_decode.prepare_workspace(q, metadata) if ( native_supported and input_type == AttentionInputType.generation_only - and 1 <= q.shape[0] <= 8 + and 1 <= num_query_tokens <= 8 ): out = self._native_decode(q, kv_rows, topk_rows, self.softmax_scale, metadata) else: diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/metadata.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/metadata.py index 279515981d55..013bdb2303d9 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/metadata.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/metadata.py @@ -75,7 +75,7 @@ def prepare(self, attn_metadata) -> None: cu.append(cu[-1] + length) self.glm_ctx_cu_seqlens = cu - width = int(getattr(manager, "max_blocks_per_seq", 0)) or 1 + width = manager.max_blocks_per_seq self._glm_ensure_tables(width) pages = manager.get_batch_slot_tables(list(request_ids)[:batch]) staging = self._glm_block_tables_cpu diff --git a/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py b/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py index 4f575ccf56c2..ca8297d61166 100644 --- a/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py +++ b/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py @@ -129,7 +129,7 @@ def glm5_next_is_quantized(model_config: ModelConfig[PretrainedConfig]) -> bool: Unquantized modules are also supported for component construction. """ - quant = getattr(model_config, "quant_config", None) + quant = model_config.quant_config if quant is None or quant.quant_algo is None: return False if quant.quant_algo != QuantAlgo.FP8_BLOCK_SCALES: diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next.py b/tensorrt_llm/_torch/models/modeling_glm5_next.py index c46e83c84a62..17cfe086604b 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next.py @@ -159,18 +159,14 @@ def glm5_next_tp_reduces(mapping: Mapping | None) -> bool: replicated attention / dense weights, so nothing is reduced there (the fused MoE does its own dispatch/combine from ``all_rank_num_tokens``). """ - return ( - mapping is not None - and int(getattr(mapping, "tp_size", 1) or 1) > 1 - and not bool(getattr(mapping, "enable_attention_dp", False)) - ) + return mapping is not None and mapping.tp_size > 1 and not mapping.enable_attention_dp def glm5_next_attention_mapping(mapping: Mapping | None) -> Mapping | None: """The Mapping the attention projections shard over: the model's, or a TP=1 view of it under attention DP (heads replicated per rank) -- the same remap :class:`~tensorrt_llm._torch.attention.mla.MLA` applies.""" - if mapping is None or not getattr(mapping, "enable_attention_dp", False): + if mapping is None or not mapping.enable_attention_dp: return mapping return Mapping( world_size=mapping.pp_size * mapping.tp_size, @@ -361,7 +357,7 @@ def build_glm5_next_runtime_context(attn_metadata: AttentionMetadata) -> Glm5Nex "glm5_next requires prepared glm_block_tables; call attn_metadata.prepare() " "with Glm5NextMamba2Metadata before eager execution or CUDA graph capture" ) - live_lengths = getattr(attn_metadata, "kv_lens_cuda", None) + live_lengths = attn_metadata.kv_lens_cuda if live_lengths is None: raise ValueError("glm5_next requires prepared attn_metadata.kv_lens_cuda") batch = int(attn_metadata.seq_lens.shape[0]) @@ -432,7 +428,7 @@ def __init__(self, model_config: ModelConfig[PretrainedConfig]) -> None: # ``model.layers.45.*`` keys are placed by the same exact loader, the # same projection swap, and the same per-owner materialization. self.mtp_layers: tuple[nn.Module, ...] = () - spec_config = getattr(model_config, "spec_config", None) + spec_config = model_config.spec_config if spec_config is not None and spec_config.spec_dec_mode.is_mtp_one_model(): mtp_layers = tuple(self.draft_model.mtp_layers) if len(mtp_layers) != 1: @@ -515,6 +511,11 @@ def get_preferred_kv_cache_manager_version(cls, pretrained_config=None) -> str: """Use V2 for the shared latent-KV, pool-indexer and recurrent-state cache.""" return "V2" + @classmethod + def get_preferred_transceiver_runtime(cls, pretrained_config=None) -> str: + """Use Python NIXL to transfer the hybrid cache, including recurrent state.""" + return "PYTHON" + # -- whole-model materialization -------------------------------------- def load_weights( @@ -712,11 +713,9 @@ def mark_linear(mod_path: str) -> None: for fused_path, parts in fused_groups.items(): dest_mod = named_modules[fused_path] groups = [parts[i] for i in sorted(parts)] - if ( - isinstance(dest_mod, Linear) - and getattr(dest_mod.weights_loading_config, "weight_mode", None) is not None - and dest_mod.weights_loading_config.weight_mode.name == ("FUSED_GATE_UP_LINEAR") - ): + if isinstance( + dest_mod, Linear + ) and dest_mod.weights_loading_config.weight_mode.name == ("FUSED_GATE_UP_LINEAR"): # GatedMLP: the Linear shards each half over this rank's # intermediate range and stacks them [gate; up] itself. dest_mod.load_weights(groups) @@ -913,8 +912,8 @@ def __init__( self.layer_idx = layer_idx self.hidden_size = int(config.hidden_size) self.total_n_heads = int(config.index_n_heads) - self.tp_size = int(getattr(mapping, "tp_size", 1) or 1) - self.tp_rank = int(getattr(mapping, "tp_rank", 0) or 0) + self.tp_size = mapping.tp_size if mapping is not None else 1 + self.tp_rank = mapping.tp_rank if mapping is not None else 0 if self.total_n_heads % self.tp_size: raise ValueError( f"glm5_next indexer has {self.total_n_heads} scoring heads, not " @@ -1008,8 +1007,8 @@ def __init__( self.layer_idx = layer_idx self.hidden_size = int(config.hidden_size) self.total_num_heads = int(config.num_attention_heads) - self.tp_size = int(getattr(attn_mapping, "tp_size", 1) or 1) - self.tp_rank = int(getattr(attn_mapping, "tp_rank", 0) or 0) + self.tp_size = attn_mapping.tp_size if attn_mapping is not None else 1 + self.tp_rank = attn_mapping.tp_rank if attn_mapping is not None else 0 if self.total_num_heads % self.tp_size: raise ValueError( f"glm5_next sparse MLA has {self.total_num_heads} heads, not divisible " @@ -1739,7 +1738,7 @@ def forward( post, comb, collapsed = self.hc_ffn.pre_mapping(hidden_states) # ADP ranks may have different phase mixes; every rank calls MoE once. mlp_out = self.run_mlp( - self.post_attention_layernorm(collapsed), getattr(metadata, "all_rank_num_tokens", None) + self.post_attention_layernorm(collapsed), metadata.all_rank_num_tokens ) return self.hc_ffn.post_mapping(mlp_out, residual, post, comb) @@ -1884,7 +1883,7 @@ def _forward_single_phase_fused_hc( norm_weight=layer.post_attention_layernorm.weight, norm_eps=layer.post_attention_layernorm.variance_epsilon, ) - mlp_out = layer.run_mlp(x, getattr(runtime_ctx.metadata, "all_rank_num_tokens", None)) + mlp_out = layer.run_mlp(x, runtime_ctx.metadata.all_rank_num_tokens) if layer_idx + 1 < num_layers: nxt = self.layers[layer_idx + 1] residual, post, comb, x = nxt.hc_attn.fused_hc( diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py index df0928009858..c326fdd375ae 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py @@ -87,7 +87,7 @@ def _image_encoder_cuda_graph_config( model_config: ModelConfig[PretrainedConfig], ) -> Optional["MultimodalEncoderCudaGraphConfig"]: - mm_config = getattr(model_config, "multimodal_config", None) + mm_config = model_config.multimodal_config if mm_config is None or mm_config.encoder_cuda_graph is None: return None unknown = set(mm_config.encoder_cuda_graph) - {"image"} @@ -109,14 +109,14 @@ def _text_dtype(model_config: ModelConfig[PretrainedConfig]) -> torch.dtype: def _require_trtllm_vision_backend(model_config: ModelConfig[PretrainedConfig], where: str) -> None: """Require full-mask TRTLLM vision attention without sparse backend wrappers.""" - backend = getattr(model_config, "attn_backend", None) + backend = model_config.attn_backend if not isinstance(backend, str) or backend.upper() != "TRTLLM": raise ValueError( f"{where}: the glm5_next vision tower supports only the TRTLLM " f"attention backend, got attn_backend={backend!r}. There is no " "VANILLA/FlashInfer/SDPA vision attention path." ) - if getattr(model_config, "sparse_attention_config", None) is not None: + if model_config.sparse_attention_config is not None: raise ValueError( f"{where}: the glm5_next vision tower runs plain full-mask TRTLLM " "attention; a sparse_attention_config would swap in a sparse " @@ -130,7 +130,7 @@ def _create_linear_weights(*modules: nn.Module) -> None: module that deferred its weights (``skip_create_weights_in_init``).""" for module in modules: for sub in module.modules(): - if isinstance(sub, Linear) and not getattr(sub, "_weights_created", True): + if isinstance(sub, Linear) and not sub._weights_created: sub.create_weights() @@ -1068,6 +1068,10 @@ def get_model_defaults(cls, llm_args) -> dict: def get_preferred_kv_cache_manager_version(cls, pretrained_config=None) -> str: return Glm5NextForCausalLM.get_preferred_kv_cache_manager_version(pretrained_config) + @classmethod + def get_preferred_transceiver_runtime(cls, pretrained_config=None) -> str: + return Glm5NextForCausalLM.get_preferred_transceiver_runtime(pretrained_config) + @property def mamba_metadata_cls(self): # The engine resolves the Mamba metadata class from the top-level model. diff --git a/tests/unittest/llmapi/test_llm_args.py b/tests/unittest/llmapi/test_llm_args.py index dca3820ed3ee..c6ff3f59907b 100644 --- a/tests/unittest/llmapi/test_llm_args.py +++ b/tests/unittest/llmapi/test_llm_args.py @@ -925,6 +925,8 @@ def test_registered_models_prefer_v2(self) -> None: "DeepseekV3ForCausalLM", "DeepseekV32ForCausalLM", "GlmMoeDsaForCausalLM", + "Glm5NextForCausalLM", + "Glm5NextForConditionalGeneration", "GptOssForCausalLM", "MistralLarge3ForCausalLM", "DeepseekV4ForCausalLM", @@ -953,7 +955,8 @@ def test_registered_models_prefer_v2(self) -> None: assert model_cls is not None assert model_cls.get_preferred_kv_cache_manager_version() == "V2" - def test_registered_models_keep_v2_on_nixl(self) -> None: + @pytest.mark.parametrize("timeout_ms", [10000, None]) + def test_registered_models_keep_v2_on_nixl(self, timeout_ms) -> None: """Models preferring V2 and the Python transceiver keep V2 on NIXL. Both sentinels start at 'auto'; production resolves the transceiver @@ -961,6 +964,9 @@ def test_registered_models_keep_v2_on_nixl(self) -> None: this list: it silently resolves to V1 on this route (its disaggregated serving is unvalidated -- the missing preference is deliberate). + + Even with an unsupported infinite timeout, the model preference must + survive resolution; transceiver creation validates the timeout later. """ from tensorrt_llm._torch.models.modeling_utils import \ get_registered_model_class @@ -970,6 +976,8 @@ def test_registered_models_keep_v2_on_nixl(self) -> None: "DeepseekV3ForCausalLM", "DeepseekV32ForCausalLM", "GlmMoeDsaForCausalLM", + "Glm5NextForCausalLM", + "Glm5NextForConditionalGeneration", "MistralLarge3ForCausalLM", "GptOssForCausalLM", "KimiK25ForConditionalGeneration", @@ -999,7 +1007,9 @@ def test_registered_models_keep_v2_on_nixl(self) -> None: llm_args = TorchLlmArgs( model="/tmp/dummy_model", cache_transceiver_config=CacheTransceiverConfig( - backend="NIXL", transceiver_runtime="auto"), + backend="NIXL", + transceiver_runtime="auto", + kv_transfer_timeout_ms=timeout_ms), ) _resolve_transceiver_runtime_auto(llm_args, model_cls) assert _resolve_kv_cache_manager_v2_auto( From 41b5db8d40dedf70f5aab5909f0c2103c90a2151 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Tue, 22 Sep 2026 20:49:12 -0700 Subject: [PATCH 23/35] [None][refactor] Validate GLM sparse metadata and forward argument types Signed-off-by: Ruocheng Jia --- .../backends/sparse/glm_kpool/backend.py | 18 ++++++------ .../backends/sparse/glm_kpool/metadata.py | 8 ++++-- .../_torch/models/modeling_glm5_next.py | 7 +++-- .../sparse/glm_kpool/test_glm_kpool.py | 28 +++++++++++++++++-- .../modeling/test_glm5_next_contracts.py | 19 +++++++++---- 5 files changed, 59 insertions(+), 21 deletions(-) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py index 16eee8b539e7..595561521822 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/backend.py @@ -42,8 +42,9 @@ ) from ...trtllm import TrtllmAttention, TrtllmAttentionMetadata from .kernels import gather_fp8_kv_rows, kpool_expand, kpool_score, kpool_update +from .metadata import Glm5NextMamba2Metadata from .native_decode import GlmKpoolNativeDecode -from .params import INDEX_SENTINEL, GlmKpoolSparseParams +from .params import INDEX_SENTINEL, GlmKpoolBackendForwardArgs, GlmKpoolSparseParams try: from tensorrt_llm.flash_mla import flash_mla_sparse_fwd @@ -230,11 +231,9 @@ def _cache_state(self, metadata) -> _GlmKpoolCacheState: "KVCacheManagerV2 owns the latent/indexer pools" ) mamba_metadata = metadata.mamba_metadata - if mamba_metadata is None or mamba_metadata is False: - raise ValueError( - "glm_kpool requires prepared metadata: call metadata.prepare() " - "with the Glm5NextCacheManager attached (mamba_metadata is missing)" - ) + assert isinstance(mamba_metadata, Glm5NextMamba2Metadata), ( + "glm_kpool requires Glm5NextMamba2Metadata" + ) latent = manager.get_latent_state_buffer(self.layer_idx) index = manager.get_index_state_buffer(self.layer_idx) if latent is None or index is None: @@ -248,7 +247,7 @@ def _cache_state(self, metadata) -> _GlmKpoolCacheState: batch = int(metadata.seq_lens.shape[0]) num_contexts = int(metadata.num_contexts) - tables = getattr(mamba_metadata, "glm_block_tables", None) + tables = mamba_metadata.glm_block_tables if tables is None: raise RuntimeError( "glm_kpool requires prepared glm_block_tables; call metadata.prepare() " @@ -555,7 +554,10 @@ def forward( """ forward_args = merge_attention_forward_args(forward_args, kwargs) sparse_args = forward_args.sparse_backend_args - topk_rows = getattr(sparse_args, "topk_rows", None) + assert isinstance(sparse_args, GlmKpoolBackendForwardArgs), ( + "glm_kpool requires GlmKpoolBackendForwardArgs" + ) + topk_rows = sparse_args.topk_rows if topk_rows is None: raise ValueError( "glm_kpool requires pool-expanded selection in sparse_backend_args.topk_rows" diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/metadata.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/metadata.py index 013bdb2303d9..98eb60cebabc 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/metadata.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/metadata.py @@ -21,6 +21,8 @@ from tensorrt_llm._torch.modules.kimi_kda.kimi_k3_mamba_metadata import KimiK3MambaMetadata from tensorrt_llm._utils import prefer_pinned +from .cache_manager import Glm5NextCacheManager + class Glm5NextMamba2Metadata(KimiK3MambaMetadata): def __init__(self, max_batch_size: int, chunk_size: int, max_num_tokens: int) -> None: @@ -47,13 +49,15 @@ def _glm_ensure_tables(self, width: int) -> None: ) def prepare(self, attn_metadata) -> None: - super().prepare(attn_metadata) manager = attn_metadata.kv_cache_manager + assert manager is None or isinstance(manager, Glm5NextCacheManager), ( + "glm5_next metadata requires Glm5NextCacheManager" + ) + super().prepare(attn_metadata) kv_params = attn_metadata.kv_cache_params request_ids = attn_metadata.request_ids if ( manager is None - or not hasattr(manager, "get_batch_slot_tables") or kv_params is None or kv_params.num_cached_tokens_per_seq is None or request_ids is None diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next.py b/tensorrt_llm/_torch/models/modeling_glm5_next.py index 17cfe086604b..6542f2b9b653 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next.py @@ -350,9 +350,10 @@ def build_glm5_next_runtime_context(attn_metadata: AttentionMetadata) -> Glm5Nex if attn_metadata.kv_cache_manager is None: raise ValueError("glm5_next requires a kv cache manager; got None") mamba_metadata = attn_metadata.mamba_metadata - if mamba_metadata is None or mamba_metadata is False: - raise ValueError("glm5_next requires mamba_metadata; call attn_metadata.prepare() first") - if getattr(mamba_metadata, "glm_block_tables", None) is None: + assert isinstance(mamba_metadata, Glm5NextMamba2Metadata), ( + "glm5_next requires Glm5NextMamba2Metadata" + ) + if mamba_metadata.glm_block_tables is None: raise RuntimeError( "glm5_next requires prepared glm_block_tables; call attn_metadata.prepare() " "with Glm5NextMamba2Metadata before eager execution or CUDA graph capture" diff --git a/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py b/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py index d008fbbe698d..526834f7fd65 100644 --- a/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py +++ b/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py @@ -25,6 +25,7 @@ AttentionInputType, ) from tensorrt_llm._torch.attention.backends.sparse.glm_kpool import ( + Glm5NextMamba2Metadata, GlmKpoolSparseAttention, GlmKpoolSparseParams, latent_pool_rows, @@ -135,6 +136,8 @@ def test_forward_dispatches_latent_rows_and_preserves_output(forward_case, route "invalid", [ "selection", + "selection_type", + "missing_rows", "legacy_indices", "v", "out_scale", @@ -155,7 +158,13 @@ def test_forward_rejects_invalid_arguments_before_cache_access(forward_case, inv error, message = ValueError, "quantized attention output" if invalid == "selection": args.sparse_backend_args = None - error, message = ValueError, "pool-expanded selection" + error, message = AssertionError, "GlmKpoolBackendForwardArgs" + elif invalid == "selection_type": + args.sparse_backend_args = SimpleNamespace(topk_rows=case.indices, topk_indices=None) + error, message = AssertionError, "GlmKpoolBackendForwardArgs" + elif invalid == "missing_rows": + args.sparse_backend_args = GlmKpoolBackendForwardArgs() + message = "pool-expanded selection" elif invalid == "legacy_indices": args.sparse_backend_args.topk_indices = case.indices message = "not request-local topk_indices" @@ -213,6 +222,8 @@ def test_cache_state_uses_prepared_metadata_without_fallback(is_cuda_graph): index = torch.zeros(4, 8, 1, 384, dtype=torch.bfloat16) tables = torch.tensor([[2, 0], [3, 1]], dtype=torch.long) live_lengths = torch.tensor([5, 8], dtype=torch.int32) + prepared = object.__new__(Glm5NextMamba2Metadata) + prepared.glm_block_tables = tables metadata = SimpleNamespace( kv_cache_manager=SimpleNamespace( tokens_per_block=8, @@ -223,7 +234,7 @@ def test_cache_state_uses_prepared_metadata_without_fallback(is_cuda_graph): num_contexts=0, is_cuda_graph=is_cuda_graph, kv_lens_cuda=live_lengths, - mamba_metadata=SimpleNamespace(glm_block_tables=tables), + mamba_metadata=prepared, ) state = backend._cache_state(metadata) assert state.block_tables.data_ptr() == tables.data_ptr() @@ -237,6 +248,19 @@ def test_cache_state_uses_prepared_metadata_without_fallback(is_cuda_graph): metadata.kv_lens_cuda = None with pytest.raises(ValueError, match="kv_lens_cuda"): backend._cache_state(metadata) + metadata.kv_lens_cuda = live_lengths + for invalid in (None, False, SimpleNamespace(glm_block_tables=tables)): + metadata.mamba_metadata = invalid + with pytest.raises(AssertionError, match="Glm5NextMamba2Metadata"): + backend._cache_state(metadata) + + +def test_metadata_prepare_rejects_non_glm_cache_manager(): + prepared = object.__new__(Glm5NextMamba2Metadata) + manager = SimpleNamespace(get_batch_slot_tables=Mock()) + with pytest.raises(AssertionError, match="Glm5NextCacheManager"): + prepared.prepare(SimpleNamespace(kv_cache_manager=manager)) + manager.get_batch_slot_tables.assert_not_called() @pytest.mark.parametrize("heads", [16, 64]) diff --git a/tests/unittest/_torch/modeling/test_glm5_next_contracts.py b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py index e037218e9e38..76f1d37a6dc0 100644 --- a/tests/unittest/_torch/modeling/test_glm5_next_contracts.py +++ b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py @@ -9,6 +9,7 @@ import torch from transformers import PretrainedConfig +from tensorrt_llm._torch.attention.backends.sparse.glm_kpool import Glm5NextMamba2Metadata from tensorrt_llm._torch.distributed import AllReduce, AllReduceStrategy from tensorrt_llm._torch.model_config import ModelConfig from tensorrt_llm._torch.models.checkpoints.hf.glm5_next_weight_mapper import ( @@ -159,11 +160,10 @@ def test_layer_masks_accept_composite_and_text_configs(): @pytest.mark.parametrize("is_cuda_graph", [False, True]) def test_runtime_context_uses_prepared_schedules_and_live_lengths(is_cuda_graph): live_lengths = torch.tensor([6, 9], dtype=torch.int32) - prepared = SimpleNamespace( - glm_block_tables=torch.tensor([[0, 1], [2, 3]]), - glm_ctx_cu_seqlens=[0, 3], - glm_cached_lens_host=[3, 5], - ) + prepared = object.__new__(Glm5NextMamba2Metadata) + prepared.glm_block_tables = torch.tensor([[0, 1], [2, 3]]) + prepared.glm_ctx_cu_seqlens = [0, 3] + prepared.glm_cached_lens_host = [3, 5] metadata = SimpleNamespace( kv_cache_manager=object(), mamba_metadata=prepared, @@ -192,6 +192,11 @@ def test_runtime_context_uses_prepared_schedules_and_live_lengths(is_cuda_graph) metadata.kv_lens_cuda = None with pytest.raises(ValueError, match="kv_lens_cuda"): build_glm5_next_runtime_context(metadata) + metadata.kv_lens_cuda = live_lengths + for invalid in (None, False, SimpleNamespace(glm_block_tables=prepared.glm_block_tables)): + metadata.mamba_metadata = invalid + with pytest.raises(AssertionError, match="Glm5NextMamba2Metadata"): + build_glm5_next_runtime_context(metadata) @pytest.mark.cpu_only @@ -450,6 +455,8 @@ def run(lengths, discard_prefix=False): ) outputs = [] cached = 0 + prepared = object.__new__(Glm5NextMamba2Metadata) + prepared.glm_block_tables = tables for length in lengths: if cached and discard_prefix: latent.zero_() @@ -459,7 +466,7 @@ def run(lengths, discard_prefix=False): seq_lens=torch.tensor([length]), num_contexts=1, kv_lens_cuda=torch.tensor([cached + length], device="cuda", dtype=torch.int32), - mamba_metadata=SimpleNamespace(glm_block_tables=tables), + mamba_metadata=prepared, ) outputs.append( layer.forward_prefill( From 71a68607167daafcdd4557d214f1d7917e1f9756 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Tue, 22 Sep 2026 21:36:44 -0700 Subject: [PATCH 24/35] [None][test] Add GLM-5.3-Flash coverage to GB300 CI Register one optimized MTP3 accuracy case pre-merge and broader post-merge/QA coverage. Add the in-tree config fallback and lazy multimodal processor initialization needed by text tests on the shared Transformers version. Signed-off-by: Ruocheng Jia --- tensorrt_llm/_torch/configs/__init__.py | 11 ++ tensorrt_llm/_torch/configs/glm5_next.py | 156 ++++++++++++++++++ .../models/modeling_glm5_next_vision.py | 27 ++- .../_torch/pyexecutor/config_utils.py | 7 + .../_torch/pyexecutor/engine/multimodal.py | 17 +- .../accuracy/test_disaggregated_serving.py | 1 + .../defs/accuracy/test_glm53_flash.py | 10 +- .../test_lists/qa/llm_function_core.txt | 13 ++ .../test_lists/test-db/l0_gb300.yml | 16 ++ .../test-db/l0_gb300_multi_gpus.yml | 5 + .../_torch/executor/engine/test_multimodal.py | 16 +- .../modeling/test_glm5_next_contracts.py | 90 +++++++++- 12 files changed, 349 insertions(+), 20 deletions(-) create mode 100644 tensorrt_llm/_torch/configs/glm5_next.py diff --git a/tensorrt_llm/_torch/configs/__init__.py b/tensorrt_llm/_torch/configs/__init__.py index dac17fdd45a9..3ad44908c691 100644 --- a/tensorrt_llm/_torch/configs/__init__.py +++ b/tensorrt_llm/_torch/configs/__init__.py @@ -23,6 +23,11 @@ Gemma4UnifiedTextConfig, Gemma4UnifiedVisionConfig, ) +from tensorrt_llm._torch.configs.glm5_next import ( + Glm5NextConfig, + Glm5NextTextConfig, + Glm5NextVisionConfig, +) from tensorrt_llm._torch.configs.k3_dspark import K3DsparkConfig from tensorrt_llm._torch.configs.kimi_k3 import KimiK3Config, KimiK3VisionConfig from tensorrt_llm._torch.configs.kimi_linear import KimiLinearConfig @@ -61,6 +66,9 @@ def _register_custom_configs_with_transformers() -> None: "deepseek_v32": DeepseekV3Config, "kimi_k2": DeepseekV3Config, "deepseek_v4": DeepseekV4Config, + "glm5_next": Glm5NextConfig, + "glm5_next_text": Glm5NextTextConfig, + "glm5_next_vision": Glm5NextVisionConfig, "gemma4_assistant": Gemma4AssistantConfig, # Kimi K3 composite multimodal config ("kimi_k3") and its text config # ("kimi_linear"). pyexecutor.config_utils.load_pretrained_config keeps @@ -103,6 +111,9 @@ def _register_custom_configs_with_transformers() -> None: "Cosmos3Config", "DeepseekV3Config", "DeepseekV4Config", + "Glm5NextConfig", + "Glm5NextTextConfig", + "Glm5NextVisionConfig", "Gemma4AssistantConfig", "Gemma4UnifiedAudioConfig", "Gemma4UnifiedConfig", diff --git a/tensorrt_llm/_torch/configs/glm5_next.py b/tensorrt_llm/_torch/configs/glm5_next.py new file mode 100644 index 000000000000..9e029e60a934 --- /dev/null +++ b/tensorrt_llm/_torch/configs/glm5_next.py @@ -0,0 +1,156 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""GLM-5.3 configuration compatibility for Transformers versions before native support. + +Image/video preprocessing still requires the native Glm5NextProcessor. +Native Transformers config classes take precedence when available. +""" + +from transformers import PretrainedConfig + +from .deepseek_v3 import DeepseekV3Config + + +class Glm5NextTextConfig(DeepseekV3Config): + model_type = "glm5_next_text" + base_config_key = "text_config" + attribute_map = {"num_local_experts": "n_routed_experts"} + + def __init__(self, **kwargs): + defaults = dict( + vocab_size=154880, + hidden_size=4096, + intermediate_size=12288, + moe_intermediate_size=2048, + num_hidden_layers=45, + num_attention_heads=64, + num_key_value_heads=64, + n_routed_experts=288, + qk_rope_head_dim=0, + qk_nope_head_dim=256, + v_head_dim=256, + n_group=1, + max_position_embeddings=1048576, + rms_norm_eps=1e-5, + pad_token_id=154820, + bos_token_id=None, + eos_token_id=None, + index_topk=2048, + index_head_dim=128, + index_n_heads=32, + index_kpool=16, + index_kpool_always_select_tail=True, + swiglu_limit=10.0, + hc_mult=4, + hc_eps=1e-6, + hc_sinkhorn_iters=20, + output_router_logits=False, + router_aux_loss_coef=0.001, + ) + fields = {**defaults, **kwargs} + # Older Transformers rejects deepseek_sparse_attention in its generic validator. + layer_types = fields.pop("layer_types", None) + super().__init__(**fields) + self.head_dim = self.qk_rope_head_dim + self.qk_head_dim = self.qk_nope_head_dim + self.qk_rope_head_dim + self.layer_types = [ + "deepseek_sparse_attention" if kind == "full_attention" else kind + for kind in ( + layer_types + if layer_types is not None + else [ + "deepseek_sparse_attention" if i % 4 == 3 else "linear_attention" + for i in range(self.num_hidden_layers) + ] + ) + ] + if getattr(self, "mlp_layer_types", None) is None: + self.mlp_layer_types = [ + "dense" if i < 3 else "sparse" for i in range(self.num_hidden_layers) + ] + if getattr(self, "indexer_types", None) is None: + pattern = kwargs.get("index_topk_pattern") + if pattern is not None: + self.indexer_types = ( + [{"F": "full", "S": "shared"}[c] for c in pattern] + if isinstance(pattern, str) + else list(pattern) + ) + else: + freq = max(kwargs.get("index_topk_freq", 1), 1) + offset = kwargs.get("index_skip_topk_offset", 2) + self.indexer_types = [ + "full" if max(i - offset + 1, 0) % freq == 0 else "shared" + for i in range(self.num_hidden_layers) + ] + linear = kwargs.get("linear_attn_config") or {} + self.linear_head_dim = linear.get("head_dim", kwargs.get("linear_head_dim", 128)) + self.linear_num_heads = linear.get("num_heads", kwargs.get("linear_num_heads", 64)) + self.linear_conv_kernel_dim = linear.get( + "short_conv_kernel_size", kwargs.get("linear_conv_kernel_dim", 4) + ) + self.linear_lower_bound = linear.get( + "gate_lower_bound", kwargs.get("linear_lower_bound", -5.0) + ) + if ( + kwargs.get("linear_attn_config") is not None + and linear.get("safe_gate", True) + and self.linear_lower_bound is None + ): + self.linear_lower_bound = -5.0 + + +class Glm5NextVisionConfig(PretrainedConfig): + model_type = "glm5_next_vision" + base_config_key = "vision_config" + attribute_map = {"num_attention_heads": "num_heads"} + + def __init__(self, **kwargs): + defaults = dict( + depth=24, + hidden_size=1024, + hidden_act="silu", + attention_bias=True, + attention_dropout=0.0, + num_heads=16, + in_channels=3, + image_size=336, + patch_size=14, + rms_norm_eps=1e-5, + spatial_merge_size=2, + temporal_patch_size=2, + out_hidden_size=1536, + intermediate_size=4096, + initializer_range=0.02, + projection_intermediate_size=10240, + swiglu_limit=10.0, + ) + super().__init__(**{**defaults, **kwargs}) + + +class Glm5NextConfig(PretrainedConfig): + model_type = "glm5_next" + sub_configs = {"text_config": Glm5NextTextConfig, "vision_config": Glm5NextVisionConfig} + keys_to_ignore_at_inference = ["past_key_values"] + + def __init__(self, text_config=None, vision_config=None, **kwargs): + self.text_config = ( + Glm5NextTextConfig(**(text_config if text_config is not None else kwargs)) + if isinstance(text_config, dict) or text_config is None + else text_config + ) + self.vision_config = ( + Glm5NextVisionConfig(**(vision_config or {})) + if isinstance(vision_config, dict) or vision_config is None + else vision_config + ) + defaults = dict( + image_token_id=154854, + video_token_id=154855, + image_start_token_id=154830, + image_end_token_id=154831, + video_start_token_id=154832, + video_end_token_id=154833, + tie_word_embeddings=False, + ) + super().__init__(**{**defaults, **kwargs}) diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py index c326fdd375ae..9b016ba56167 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py @@ -799,9 +799,7 @@ def __init__( self._tokenizer = ( tokenizer if tokenizer is not None else AutoTokenizer.from_pretrained(model_path) ) - self._processor = AutoProcessor.from_pretrained( - model_path, use_fast=self._use_fast, trust_remote_code=trust_remote_code - ) + self._processor = None vision = config.vision_config self._merge_size = int(vision.spatial_merge_size) self._patch_size = int(vision.patch_size) @@ -820,6 +818,17 @@ def model_path(self) -> str: @property def processor(self) -> AutoProcessor: + if self._processor is None: + from transformers.models.auto.processing_auto import PROCESSOR_MAPPING_NAMES + + if "glm5_next" not in PROCESSOR_MAPPING_NAMES: + raise RuntimeError( + "GLM-5.3-Flash image/video processing requires transformers==5.17.0. " + "For text-only inference on older Transformers, set disable_mm_encoder=True." + ) + self._processor = AutoProcessor.from_pretrained( + self.model_path, use_fast=self._use_fast, trust_remote_code=self._trust_remote_code + ) return self._processor @property @@ -873,7 +882,7 @@ def get_num_tokens_per_video( if video_grid_thw is None: meta = dict(video_metadata or {}) meta["total_num_frames"] = len(video) - video_grid_thw = self._processor.video_processor( + video_grid_thw = self.processor.video_processor( videos=[video], video_metadata=[meta], do_sample_frames=False, return_tensors="pt" )["video_grid_thw"] grid = torch.as_tensor(video_grid_thw) @@ -889,7 +898,7 @@ def _max_grid_side(self, max_patches: int) -> int: def _processor_max_patches(self) -> int: """HF's image-token budget expressed in pre-merge patch rows.""" - processor = self._processor.image_processor + processor = self.processor.image_processor return int(processor.max_image_tokens) * int(processor.merge_size) ** 2 def get_mm_max_tokens_per_item( @@ -915,7 +924,7 @@ def get_dummy_mm_data( side = self._max_grid_side(min(per_item, self._processor_max_patches())) pixels_side = side * self._patch_size image = Image.new("RGB", (pixels_side, pixels_side), (127, 127, 127)) - out = self._processor.image_processor(images=[image] * num_images, return_tensors="pt") + out = self.processor.image_processor(images=[image] * num_images, return_tensors="pt") return { "image": { "pixel_values": out["pixel_values"].to(dtype or self._dtype), @@ -964,8 +973,8 @@ def call_with_text_prompt( # mismatch must surface as a ValueError (an HTTP error at the serving # boundary), not as the HF processor's iterator exhaustion. for placeholder, items, what in ( - (self._processor.image_token, images or [], "image"), - (self._processor.video_token, video_datas or [], "video"), + (self.processor.image_token, images or [], "image"), + (self.processor.video_token, video_datas or [], "video"), ): n_placeholders = text_prompt.count(placeholder) if n_placeholders != len(items): @@ -975,7 +984,7 @@ def call_with_text_prompt( "refusing the mismatched request" ) - processed = self._processor( + processed = self.processor( text=[text_prompt], images=images, videos=videos, diff --git a/tensorrt_llm/_torch/pyexecutor/config_utils.py b/tensorrt_llm/_torch/pyexecutor/config_utils.py index 37ec53db3fe7..d0ad17384404 100644 --- a/tensorrt_llm/_torch/pyexecutor/config_utils.py +++ b/tensorrt_llm/_torch/pyexecutor/config_utils.py @@ -1030,6 +1030,13 @@ def load_pretrained_config(model_name_or_path: str, model_config.text_config = transformers.Qwen3NextConfig.from_dict( Qwen35ConfigCompat.normalize(config_dict, require_text_config=True)) _normalize_qwen35_quantization_config(model_config) + elif model_type in ("glm5_next", "glm5_next_text"): + from tensorrt_llm._torch import configs + config_name = ("Glm5NextConfig" + if model_type == "glm5_next" else "Glm5NextTextConfig") + config_class = getattr(transformers, config_name, + getattr(configs, config_name)) + model_config = config_class.from_dict(config_dict, **kwargs) elif model_type == "glm_moe_dsa": # GLM-MoE-DSA configs tag every layer with # layer_types=['deepseek_sparse_attention', ...] for HF bookkeeping. diff --git a/tensorrt_llm/_torch/pyexecutor/engine/multimodal.py b/tensorrt_llm/_torch/pyexecutor/engine/multimodal.py index 9effb6644d30..963b9cd33786 100644 --- a/tensorrt_llm/_torch/pyexecutor/engine/multimodal.py +++ b/tensorrt_llm/_torch/pyexecutor/engine/multimodal.py @@ -133,18 +133,21 @@ def setup_mm_encoder_attn_metadata( function rather than a ``MultimodalItemScheduler`` method: the scheduler is ``None`` whenever item scheduling is off. """ + encoders = [module for module in model.modules() if isinstance(module, MultimodalEncoderMixin)] + if not encoders: + return + max_seq_len = encoder_max_num_tokens if isinstance(input_processor, BaseMultimodalDummyInputsBuilder): max_tokens_per_item = input_processor.get_mm_max_tokens_per_item() max_seq_len = max(max_seq_len, max(max_tokens_per_item.values(), default=0)) - for module in model.modules(): - if isinstance(module, MultimodalEncoderMixin): - setup_kwargs: dict[str, Any] = dict(max_num_tokens=encoder_max_num_tokens) - if attention_metadata_capacity is not None: - setup_kwargs["attention_metadata_capacity"] = attention_metadata_capacity - module.setup_attn_metadata(**setup_kwargs) - module.set_attn_max_seq_len(max_seq_len) + for module in encoders: + setup_kwargs: dict[str, Any] = dict(max_num_tokens=encoder_max_num_tokens) + if attention_metadata_capacity is not None: + setup_kwargs["attention_metadata_capacity"] = attention_metadata_capacity + module.setup_attn_metadata(**setup_kwargs) + module.set_attn_max_seq_len(max_seq_len) def resolve_bytes_per_mm_encoder_embedding(model: MultimodalModelMixin) -> int: diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index d5ebfd1e9aae..25b7d70c4f3c 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -2277,6 +2277,7 @@ def test_fp8_nixl(self, mtp): "tensor_parallel_size": 4, "pipeline_parallel_size": 1, "moe_expert_parallel_size": 4, + "disable_mm_encoder": True, "max_batch_size": 64, "max_num_tokens": 16384, "max_seq_len": 8192, diff --git a/tests/integration/defs/accuracy/test_glm53_flash.py b/tests/integration/defs/accuracy/test_glm53_flash.py index 7be614093059..d2c99fecd107 100644 --- a/tests/integration/defs/accuracy/test_glm53_flash.py +++ b/tests/integration/defs/accuracy/test_glm53_flash.py @@ -40,7 +40,7 @@ class TestGLM53FlashFP8(LlmapiAccuracyTestHarness): - """GLM-5.3-Flash FP8 accuracy and runtime tests on B200. + """GLM-5.3-Flash FP8 accuracy and runtime tests on Blackwell. Block reuse is enabled only in the periodic-snapshot test. """ @@ -60,6 +60,8 @@ def _llm_kwargs(tp_size: int, ep_size: int) -> dict: max_seq_len=8192, cuda_graph_config=CudaGraphConfig(max_batch_size=64, enable_padding=True), disable_overlap_scheduler=False, + enable_chunked_prefill=True, + disable_mm_encoder=True, ) @staticmethod @@ -68,6 +70,7 @@ def _assert_glm5_next_stack(llm: LLM) -> None: assert llm.args.kv_cache_config.enable_block_reuse is False assert llm.args.cuda_graph_config is not None assert llm.args.cuda_graph_config.enable_padding is True + assert llm.args.disable_overlap_scheduler is False @skip_pre_blackwell @pytest.mark.skip_less_mpi_world_size(4) @@ -157,6 +160,7 @@ def _assert_kv_cache_reuse(llm: LLM) -> None: @parametrize_with_ids("tp_size,ep_size", [(4, 4)]) def test_mmmu(self, tp_size, ep_size): kwargs = self._llm_kwargs(tp_size, ep_size) + kwargs["disable_mm_encoder"] = False # MMMU prompts fit in 8K (MAX_INPUT_LEN); a smaller token budget and # batch keep the run cheap. Profiling peaks are the same as the text # tests' (about 92 GiB per GPU at 16K / batch 64 on B200). @@ -278,6 +282,7 @@ def test_mtp(self, tp_size, ep_size): ) as llm: self._assert_glm5_next_stack(llm) assert llm.args.speculative_config.max_draft_len == 3 + assert llm.args.enable_chunked_prefill is True task = GSM8K(self.MODEL_NAME) task.evaluate(llm) assert_acceptance_length_for_llm("TestGLM53FlashFP8::test_mtp", llm) @@ -290,7 +295,7 @@ def test_mtp(self, tp_size, ep_size): ids=["adp-mtp3", "tp-fp8-kv", "adp-mtp3-fp8-kv"], ) def test_attention_dp_mtp_and_fp8_kv(self, attention_dp, draft_len, cache_dtype): - """Manual regressions for the supported attention-DP, MTP and KV-cache combinations.""" + """Regressions for the supported attention-DP, MTP and KV-cache combinations.""" kwargs = self._llm_kwargs(4, 4) kwargs.update( enable_attention_dp=attention_dp, @@ -337,6 +342,7 @@ def test_runtime(self) -> None: disable_overlap_scheduler=True, enable_autotuner=False, return_perf_metrics=True, + disable_mm_encoder=True, ) as llm: assert llm.args.enable_chunked_prefill is True assert llm.args.max_num_tokens == max_num_tokens diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 798a9e4da53c..1a2c9a8bfa0f 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -16,6 +16,8 @@ accuracy/test_disaggregated_serving.py::TestDeepSeekV4Flash::test_auto_dtype accuracy/test_disaggregated_serving.py::TestDeepSeekV4Flash::test_gen_first accuracy/test_disaggregated_serving.py::TestDeepSeekV4FlashBase::test_auto_dtype accuracy/test_disaggregated_serving.py::TestGLM52NVFP4::test_nvfp4_nixl[cache_mgr_v1] +accuracy/test_disaggregated_serving.py::TestGLM53FlashFP8::test_fp8_nixl[mtp=False] +accuracy/test_disaggregated_serving.py::TestGLM53FlashFP8::test_fp8_nixl[mtp=True] accuracy/test_disaggregated_serving.py::TestGPTOSS20B::test_beam_search accuracy/test_disaggregated_serving.py::TestGPTOSS::test_auto_dtype[False] accuracy/test_disaggregated_serving.py::TestGPTOSS::test_auto_dtype[True] @@ -59,6 +61,17 @@ accuracy/test_glm52.py::TestGLM52NVFP4::test_runtime accuracy/test_glm52.py::TestGLM52NVFP4::test_tep[tp_size=8-ep_size=8] accuracy/test_glm52.py::TestGLM52NVFP4::test_tep_nvfp4kv[tp_size=8-ep_size=8] accuracy/test_glm52.py::TestGLM5FP8::test_8gpus[tp_size=8-ep_size=8] +accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_attention_dp[tp_size=4-ep_size=4] +accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_attention_dp_mtp_and_fp8_kv[adp-mtp3] +accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_attention_dp_mtp_and_fp8_kv[adp-mtp3-fp8-kv] +accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_attention_dp_mtp_and_fp8_kv[tp-fp8-kv] +accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_block_reuse[tp_size=4-ep_size=4] +accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_chunked_prefill_parity +accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_mmmu[tp_size=4-ep_size=4] +accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_mtp[tp_size=4-ep_size=4] +accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_runtime +accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_tep[tp_size=4-ep_size=4] +accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_video_url accuracy/test_kimi3.py::TestKimiK3DSpark::test_gsm8k_tep8 accuracy/test_kv_pool_rebalance_accuracy.py::TestKvPoolRebalanceAccuracy::test_rebalance_matches_baseline[no_overlap] accuracy/test_kv_pool_rebalance_accuracy.py::TestKvPoolRebalanceAccuracy::test_rebalance_matches_baseline[overlap] diff --git a/tests/integration/test_lists/test-db/l0_gb300.yml b/tests/integration/test_lists/test-db/l0_gb300.yml index 40c08cb821a6..37629950c7c5 100644 --- a/tests/integration/test_lists/test-db/l0_gb300.yml +++ b/tests/integration/test_lists/test-db/l0_gb300.yml @@ -26,3 +26,19 @@ l0_gb300: - accuracy/test_disaggregated_serving.py::TestQwen3_8_Flash_Next::test_fp8_nixl_python[prefix_cache] - unittest/_torch/thop/parallel TIMEOUT (90) - unittest/_torch/visual_gen/kernels/parallel +- condition: + ranges: + system_gpu_count: + gte: 4 + lte: 4 + wildcards: + gpu: + - '*gb110*' + - '*gb300*' + linux_distribution_name: ubuntu* + cpu: aarch64 + terms: + stage: pre_merge + backend: pytorch + tests: + - accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_mtp[tp_size=4-ep_size=4] TIMEOUT (60) diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml index 25474411ecb9..60550f612078 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml @@ -34,6 +34,11 @@ l0_gb300_multi_gpus: - unittest/_torch/thop/serial - unittest/_torch/executor - unittest/_torch/disaggregation + # GLM-5.3-Flash: non-speculative, ADP/MTP/FP8-KV, reuse and eager/chunked paths. + - accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_tep[tp_size=4-ep_size=4] TIMEOUT (60) + - accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_attention_dp_mtp_and_fp8_kv[adp-mtp3-fp8-kv] TIMEOUT (60) + - accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_block_reuse[tp_size=4-ep_size=4] TIMEOUT (60) + - accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_runtime # ------------- modules (non-MoE) --------------- - unittest/_torch/modules/test_fused_add_rms_norm_quant.py - unittest/_torch/modules/test_fused_activation_quant.py diff --git a/tests/unittest/_torch/executor/engine/test_multimodal.py b/tests/unittest/_torch/executor/engine/test_multimodal.py index 4abd6fe31d5c..635ab63c5802 100644 --- a/tests/unittest/_torch/executor/engine/test_multimodal.py +++ b/tests/unittest/_torch/executor/engine/test_multimodal.py @@ -4,6 +4,7 @@ from collections.abc import Sequence from types import SimpleNamespace from typing import Any +from unittest.mock import Mock import pytest import torch @@ -20,6 +21,7 @@ from tensorrt_llm._torch.pyexecutor.engine.multimodal import ( MultimodalItemScheduler, resolve_mm_encoder_output_budget, + setup_mm_encoder_attn_metadata, validate_mm_encoder_scheduling_compatibility, ) from tensorrt_llm._torch.pyexecutor.llm_request import ( @@ -31,13 +33,25 @@ from tensorrt_llm._torch.tensor_lru_cache import TensorLRUCache from tensorrt_llm.bindings import SamplingConfig from tensorrt_llm.inputs.multimodal import MULTIMODAL_ENCODER_ITEM_METADATA_KEY, MultimodalParams -from tensorrt_llm.inputs.registry import MultimodalEncoderItemMetadata +from tensorrt_llm.inputs.registry import ( + BaseMultimodalDummyInputsBuilder, + MultimodalEncoderItemMetadata, +) from tensorrt_llm.llmapi.llm_args import MultimodalEncoderSchedulingPolicy # The item-scheduling surface is pure logic: no kernels, no device transfers. pytestmark = pytest.mark.cpu_only +def test_disabled_encoder_does_not_initialize_processor(): + processor = Mock(spec=BaseMultimodalDummyInputsBuilder) + processor.get_mm_max_tokens_per_item.side_effect = AssertionError("encoder is disabled") + model = torch.nn.Module() + model.mm_encoder = None + setup_mm_encoder_attn_metadata(model, processor, 1024, None) + processor.get_mm_max_tokens_per_item.assert_not_called() + + def _cache_request( request_id: int, *, diff --git a/tests/unittest/_torch/modeling/test_glm5_next_contracts.py b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py index 76f1d37a6dc0..461eb8e2db2b 100644 --- a/tests/unittest/_torch/modeling/test_glm5_next_contracts.py +++ b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py @@ -7,7 +7,7 @@ import pytest import torch -from transformers import PretrainedConfig +from transformers import AutoConfig, PretrainedConfig from tensorrt_llm._torch.attention.backends.sparse.glm_kpool import Glm5NextMamba2Metadata from tensorrt_llm._torch.distributed import AllReduce, AllReduceStrategy @@ -35,6 +35,94 @@ from tensorrt_llm.mapping import Mapping +@pytest.mark.cpu_only +def test_config_fallback_round_trip(tmp_path): + from transformers.models.auto.configuration_auto import CONFIG_MAPPING + + from tensorrt_llm._torch.configs.glm5_next import Glm5NextConfig, Glm5NextTextConfig + from tensorrt_llm._torch.pyexecutor.config_utils import load_pretrained_config + + fields = _config().to_dict() + fields["architectures"] = ["Glm5NextForConditionalGeneration"] + fields["quantization_config"] = {"quant_method": "fp8", "weight_block_size": [128, 128]} + config = Glm5NextConfig.from_dict(fields) + config.save_pretrained(tmp_path) + loaded = load_pretrained_config(str(tmp_path)) + assert isinstance(loaded, CONFIG_MAPPING["glm5_next"]) + assert isinstance(AutoConfig.from_pretrained(tmp_path), type(loaded)) + assert loaded.text_config.layer_types == fields["text_config"]["layer_types"] + assert loaded.text_config.linear_attn_config == fields["text_config"]["linear_attn_config"] + assert loaded.text_config.qk_head_dim == 256 + assert loaded.text_config.head_dim == 0 + assert loaded.vision_config.spatial_merge_size == 2 + assert loaded.quantization_config == fields["quantization_config"] + assert get_glm5_next_layer_masks(loaded) == ([False, True], [True, False]) + # Registration must prefer native classes when the installed Transformers has them. + if "transformers.models.glm5_next" in type(loaded).__module__: + assert type(loaded) is not Glm5NextConfig + else: + assert type(loaded.text_config) is Glm5NextTextConfig + + +@pytest.mark.cpu_only +def test_config_fallback_matches_native(): + from tensorrt_llm._torch.configs.glm5_next import Glm5NextConfig + + native = pytest.importorskip("transformers.models.glm5_next.configuration_glm5_next") + fields = _config().to_dict() + fallback = Glm5NextConfig.from_dict(fields) + reference = native.Glm5NextConfig.from_dict(fields) + for key in ( + "num_hidden_layers", + "num_attention_heads", + "num_key_value_heads", + "hidden_size", + "q_lora_rank", + "kv_lora_rank", + "qk_head_dim", + "head_dim", + "v_head_dim", + "index_topk", + "index_kpool", + "indexer_types", + "linear_attn_config", + "linear_head_dim", + "linear_num_heads", + "linear_conv_kernel_dim", + "linear_lower_bound", + "layer_types", + "mlp_layer_types", + "hc_mult", + "hc_eps", + "hc_sinkhorn_iters", + "swiglu_limit", + ): + assert getattr(fallback.text_config, key) == getattr(reference.text_config, key), key + for key in ("depth", "num_heads", "hidden_size", "spatial_merge_size", "patch_size"): + assert getattr(fallback.vision_config, key) == getattr(reference.vision_config, key), key + + +@pytest.mark.cpu_only +def test_text_processing_without_native_processor(monkeypatch): + from transformers.models.auto.processing_auto import PROCESSOR_MAPPING_NAMES + + from tensorrt_llm._torch.models.modeling_glm5_next_vision import Glm5NextInputProcessor + from tensorrt_llm.llmapi import SamplingParams + + monkeypatch.delitem(PROCESSOR_MAPPING_NAMES, "glm5_next", raising=False) + tokenizer = Mock(return_value=SimpleNamespace(input_ids=torch.tensor([[1, 2, 3]]))) + with patch( + "tensorrt_llm._torch.models.modeling_glm5_next_vision.AutoProcessor.from_pretrained" + ) as load_processor: + processor = Glm5NextInputProcessor("unused", _config(), tokenizer=tokenizer) + tokens, extra = processor.call_with_text_prompt({"prompt": "hello"}, SamplingParams()) + assert tokens == [1, 2, 3] + assert extra is None + load_processor.assert_not_called() + with pytest.raises(RuntimeError, match="transformers==5.17.0"): + processor.get_mm_max_tokens_per_item() + + def _config(): config = PretrainedConfig( model_type="glm5_next", From 24fe42adfe888192fbb78527e165837024d1c4bd Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Wed, 23 Sep 2026 02:21:21 -0700 Subject: [PATCH 25/35] [None][fix] Separate GLM image and video preprocessing settings Signed-off-by: Ruocheng Jia --- .../models/modeling_glm5_next_vision.py | 18 +-- .../modeling/test_glm5_next_contracts.py | 110 ++++++++++++++++++ 2 files changed, 121 insertions(+), 7 deletions(-) diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py index 9b016ba56167..98657afc3cb2 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py @@ -950,15 +950,16 @@ def call_with_text_prompt( # Media loaders may deliver pre-rescaled float tensors (the server # default) instead of PIL / uint8; tell the HF processor not to # rescale those a second time (Qwen2-VL precedent). - do_rescale = True - if images and isinstance(images[0], torch.Tensor): - do_rescale = False + images_kwargs = dict(mm_processor_kwargs.get("images_kwargs") or {}) + videos_kwargs = dict(mm_processor_kwargs.get("videos_kwargs") or {}) + if images and "do_rescale" not in mm_processor_kwargs: + images_kwargs.setdefault("do_rescale", not isinstance(images[0], torch.Tensor)) videos = None video_metadata = None if video_datas: videos = [video_data.frames for video_data in video_datas] - if isinstance(videos[0][0], torch.Tensor): - do_rescale = False + if "do_rescale" not in mm_processor_kwargs: + videos_kwargs.setdefault("do_rescale", not isinstance(videos[0][0], torch.Tensor)) # Frames are already sampled by the media loader; the processor # only needs fps / frame indices to lay out the per-frame # timestamps (``VideoMetadata.timestamps``). @@ -967,7 +968,11 @@ def call_with_text_prompt( meta = dict(video_data.metadata or {}) meta["total_num_frames"] = len(video_data.frames) video_metadata.append(meta) - mm_processor_kwargs.setdefault("do_sample_frames", False) + if "do_sample_frames" not in mm_processor_kwargs: + videos_kwargs.setdefault("do_sample_frames", False) + + mm_processor_kwargs["images_kwargs"] = images_kwargs + mm_processor_kwargs["videos_kwargs"] = videos_kwargs # Fail closed before any pixel work: a placeholder / item count # mismatch must surface as a ValueError (an HTTP error at the serving @@ -989,7 +994,6 @@ def call_with_text_prompt( images=images, videos=videos, video_metadata=video_metadata, - do_rescale=do_rescale, return_tensors="pt", **mm_processor_kwargs, ) diff --git a/tests/unittest/_torch/modeling/test_glm5_next_contracts.py b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py index 461eb8e2db2b..33804f2195fa 100644 --- a/tests/unittest/_torch/modeling/test_glm5_next_contracts.py +++ b/tests/unittest/_torch/modeling/test_glm5_next_contracts.py @@ -2,6 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 """GLM configuration, loading and vision ownership regressions without checkpoint files.""" +from copy import deepcopy from types import MethodType, SimpleNamespace from unittest.mock import Mock, create_autospec, patch @@ -123,6 +124,115 @@ def test_text_processing_without_native_processor(monkeypatch): processor.get_mm_max_tokens_per_item() +@pytest.mark.cpu_only +@pytest.mark.parametrize( + "overrides,image_rescale,video_rescale,sample_frames", + [ + ({}, True, False, False), + ({"do_rescale": True, "do_sample_frames": True}, True, True, True), + ( + { + "images_kwargs": {"do_rescale": False}, + "videos_kwargs": {"do_rescale": True, "do_sample_frames": True}, + }, + False, + True, + True, + ), + ], + ids=["defaults", "flat-overrides", "modality-overrides"], +) +def test_mixed_processor_kwargs(overrides, image_rescale, video_rescale, sample_frames): + from PIL import Image + + from tensorrt_llm._torch.models.modeling_glm5_next_vision import Glm5NextInputProcessor + from tensorrt_llm.inputs.multimodal_data import VideoData + from tensorrt_llm.llmapi import SamplingParams + + processor = Glm5NextInputProcessor("unused", _config(), tokenizer=Mock()) + processor._processor = Mock( + image_token="<|image|>", + video_token="<|video|>", + return_value={"input_ids": torch.tensor([[1, 2]])}, + ) + original = deepcopy(overrides) + processor.call_with_text_prompt( + { + "prompt": "<|image|><|video|>", + "multi_modal_data": { + "image": [Image.new("RGB", (28, 28))], + "video": [VideoData(frames=[torch.zeros(3, 28, 28)] * 2, metadata={})], + }, + "mm_processor_kwargs": overrides, + }, + SamplingParams(), + ) + kwargs = processor._processor.call_args.kwargs + assert kwargs["images_kwargs"].get("do_rescale", kwargs.get("do_rescale")) == image_rescale + assert kwargs["videos_kwargs"].get("do_rescale", kwargs.get("do_rescale")) == video_rescale + assert ( + kwargs["videos_kwargs"].get("do_sample_frames", kwargs.get("do_sample_frames")) + == sample_frames + ) + assert overrides == original + assert kwargs["images_kwargs"] is not overrides.get("images_kwargs") + assert kwargs["videos_kwargs"] is not overrides.get("videos_kwargs") + + +@pytest.mark.cpu_only +@pytest.mark.parametrize("tensor_images", [False, True], ids=["pil-image", "tensor-image"]) +def test_mixed_processor_matches_separate_inputs(tensor_images): + native = pytest.importorskip("transformers.models.glm5_next.processing_glm5_next") + from PIL import Image + from tokenizers import Tokenizer + from tokenizers.models import WordLevel + from transformers import Glm5NextImageProcessor, Glm5NextVideoProcessor, PreTrainedTokenizerFast + + from tensorrt_llm._torch.models.modeling_glm5_next_vision import Glm5NextInputProcessor + from tensorrt_llm.inputs.multimodal_data import VideoData + from tensorrt_llm.llmapi import SamplingParams + + special_tokens = ["[UNK]", "<|image|>", "<|video|>", "<|begin_of_video|>", "<|end_of_video|>"] + tokenizer = PreTrainedTokenizerFast( + tokenizer_object=Tokenizer( + WordLevel(dict(zip(special_tokens, range(5))), unk_token="[UNK]") + ), + unk_token="[UNK]", + additional_special_tokens=special_tokens[1:], + ) + processor = Glm5NextInputProcessor("unused", _config(), tokenizer=tokenizer) + processor._processor = native.Glm5NextProcessor( + image_processor=Glm5NextImageProcessor(do_resize=False), + video_processor=Glm5NextVideoProcessor(do_resize=False), + tokenizer=tokenizer, + ) + pil_image = Image.new("RGB", (28, 28), (64, 128, 192)) + tensor_image = ( + torch.tensor([64, 128, 192], dtype=torch.float32)[:, None, None].expand(3, 28, 28) / 255 + ) + media = { + "image": [tensor_image if tensor_images else pil_image], + "video": [ + VideoData( + frames=[pil_image if tensor_images else tensor_image] * 2, + metadata={"fps": 2, "frames_indices": [0, 1]}, + ) + ], + } + _, mixed = processor.call_with_text_prompt( + {"prompt": "<|image|><|video|>", "multi_modal_data": media}, SamplingParams() + ) + for modality in ("image", "video"): + _, separate = processor.call_with_text_prompt( + {"prompt": f"<|{modality}|>", "multi_modal_data": {modality: media[modality]}}, + SamplingParams(), + ) + for key, expected in separate["multimodal_data"][modality].items(): + torch.testing.assert_close( + mixed["multimodal_data"][modality][key], expected, rtol=0, atol=0 + ) + + def _config(): config = PretrainedConfig( model_type="glm5_next", From 2ce830420c35d38c97f16cd31623fe6aa95d54e3 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Wed, 23 Sep 2026 02:30:01 -0700 Subject: [PATCH 26/35] [None][test] Reduce GLM-5.3-Flash end-to-end CI coverage Signed-off-by: Ruocheng Jia --- .../accuracy/test_disaggregated_serving.py | 88 ------------------- .../test_lists/qa/llm_function_core.txt | 9 -- .../test-db/l0_gb300_multi_gpus.yml | 4 +- 3 files changed, 1 insertion(+), 100 deletions(-) diff --git a/tests/integration/defs/accuracy/test_disaggregated_serving.py b/tests/integration/defs/accuracy/test_disaggregated_serving.py index 25b7d70c4f3c..d6f8287c998e 100644 --- a/tests/integration/defs/accuracy/test_disaggregated_serving.py +++ b/tests/integration/defs/accuracy/test_disaggregated_serving.py @@ -2235,94 +2235,6 @@ def test_fp8_nixl_python(self, mocker, snapshot_policy): extra_evaluator_kwargs={GSM8K: self.GSM8K_EVALUATOR_KWARGS}) -@pytest.mark.timeout(DEFAULT_TEST_TIMEOUT) -@skip_pre_blackwell -@pytest.mark.skip_less_device_memory(80000) -class TestGLM53FlashFP8(LlmapiAccuracyTestHarness): - """GLM-5.3-Flash (glm5_next): hybrid KDA + sparse-MLA layers. - - Only the Python NIXL transceiver moves the recurrent state and the indexer - side cache between workers, so the C++ transceiver is not an option here. - """ - MODEL_NAME = "zai-org/GLM-5.3-Flash" - MODEL_PATH = f"{llm_models_root()}/GLM-5.3-Flash" - - @pytest.mark.skip_less_device(8) - @parametrize_with_ids("mtp", [False, True]) - def test_fp8_nixl(self, mtp): - # Import the selected agent, including the automatic Python fallback - # when C++ bindings are unavailable. A Python-package-only check - # would incorrectly skip builds that use the C++ agent. - pytest.importorskip( - "tensorrt_llm._torch.disaggregation.nixl.agent", - reason="selected NIXL transfer agent is unavailable", - exc_type=ImportError, - ) - kv_cache_config = { - "free_gpu_memory_fraction": 0.5, - "enable_block_reuse": False, - } - cache_transceiver_config = { - "backend": "NIXL", - "transceiver_runtime": "PYTHON", - } - # The checkpoint's single MTP layer chained for three drafts; both - # workers carry it so the context worker hands over a replay-ready - # KDA state and the generation worker verifies on the fused kernel. - speculative_config = { - "decoding_type": "MTP", - "max_draft_len": 3, - } if mtp else None - common_config = { - "tensor_parallel_size": 4, - "pipeline_parallel_size": 1, - "moe_expert_parallel_size": 4, - "disable_mm_encoder": True, - "max_batch_size": 64, - "max_num_tokens": 16384, - "max_seq_len": 8192, - "kv_cache_config": kv_cache_config, - "speculative_config": speculative_config, - "cache_transceiver_config": cache_transceiver_config, - } - ctx_server_config = { - **common_config, - "disable_overlap_scheduler": True, - "cuda_graph_config": None, - } - gen_server_config = { - **common_config, - "disable_overlap_scheduler": False, - "cuda_graph_config": { - "max_batch_size": 64, - "enable_padding": True, - }, - } - disaggregated_server_config = { - "hostname": "localhost", - "backend": "pytorch", - "context_servers": { - "num_instances": 1 - }, - "generation_servers": { - "num_instances": 1 - } - } - with launch_disaggregated_llm(disaggregated_server_config, - ctx_server_config, - gen_server_config, - self.MODEL_PATH, - max_workers=64) as llm: - # launch_disaggregated_llm builds a bare LlmArgs for the DuckLLM; - # fill in the quantization so the reference lookup matches the - # registered FP8_BLOCK_SCALES entry. - llm.args.quant_config.quant_algo = "FP8_BLOCK_SCALES" - if mtp: - llm.args.speculative_config = MTPDecodingConfig( - max_draft_len=speculative_config["max_draft_len"]) - run_accuracy_test(llm, self.MODEL_NAME, ["GSM8K"]) - - @pytest.mark.timeout(DEFAULT_TEST_TIMEOUT) @skip_pre_blackwell @pytest.mark.skip_less_device_memory(80000) diff --git a/tests/integration/test_lists/qa/llm_function_core.txt b/tests/integration/test_lists/qa/llm_function_core.txt index 1a2c9a8bfa0f..3fd283ec4825 100644 --- a/tests/integration/test_lists/qa/llm_function_core.txt +++ b/tests/integration/test_lists/qa/llm_function_core.txt @@ -16,8 +16,6 @@ accuracy/test_disaggregated_serving.py::TestDeepSeekV4Flash::test_auto_dtype accuracy/test_disaggregated_serving.py::TestDeepSeekV4Flash::test_gen_first accuracy/test_disaggregated_serving.py::TestDeepSeekV4FlashBase::test_auto_dtype accuracy/test_disaggregated_serving.py::TestGLM52NVFP4::test_nvfp4_nixl[cache_mgr_v1] -accuracy/test_disaggregated_serving.py::TestGLM53FlashFP8::test_fp8_nixl[mtp=False] -accuracy/test_disaggregated_serving.py::TestGLM53FlashFP8::test_fp8_nixl[mtp=True] accuracy/test_disaggregated_serving.py::TestGPTOSS20B::test_beam_search accuracy/test_disaggregated_serving.py::TestGPTOSS::test_auto_dtype[False] accuracy/test_disaggregated_serving.py::TestGPTOSS::test_auto_dtype[True] @@ -61,17 +59,10 @@ accuracy/test_glm52.py::TestGLM52NVFP4::test_runtime accuracy/test_glm52.py::TestGLM52NVFP4::test_tep[tp_size=8-ep_size=8] accuracy/test_glm52.py::TestGLM52NVFP4::test_tep_nvfp4kv[tp_size=8-ep_size=8] accuracy/test_glm52.py::TestGLM5FP8::test_8gpus[tp_size=8-ep_size=8] -accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_attention_dp[tp_size=4-ep_size=4] -accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_attention_dp_mtp_and_fp8_kv[adp-mtp3] accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_attention_dp_mtp_and_fp8_kv[adp-mtp3-fp8-kv] -accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_attention_dp_mtp_and_fp8_kv[tp-fp8-kv] accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_block_reuse[tp_size=4-ep_size=4] -accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_chunked_prefill_parity accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_mmmu[tp_size=4-ep_size=4] accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_mtp[tp_size=4-ep_size=4] -accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_runtime -accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_tep[tp_size=4-ep_size=4] -accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_video_url accuracy/test_kimi3.py::TestKimiK3DSpark::test_gsm8k_tep8 accuracy/test_kv_pool_rebalance_accuracy.py::TestKvPoolRebalanceAccuracy::test_rebalance_matches_baseline[no_overlap] accuracy/test_kv_pool_rebalance_accuracy.py::TestKvPoolRebalanceAccuracy::test_rebalance_matches_baseline[overlap] diff --git a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml index 60550f612078..c25b2148d14d 100644 --- a/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml +++ b/tests/integration/test_lists/test-db/l0_gb300_multi_gpus.yml @@ -34,11 +34,9 @@ l0_gb300_multi_gpus: - unittest/_torch/thop/serial - unittest/_torch/executor - unittest/_torch/disaggregation - # GLM-5.3-Flash: non-speculative, ADP/MTP/FP8-KV, reuse and eager/chunked paths. - - accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_tep[tp_size=4-ep_size=4] TIMEOUT (60) + # GLM-5.3-Flash: ADP/MTP/FP8-KV and non-speculative prefix reuse. - accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_attention_dp_mtp_and_fp8_kv[adp-mtp3-fp8-kv] TIMEOUT (60) - accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_block_reuse[tp_size=4-ep_size=4] TIMEOUT (60) - - accuracy/test_glm53_flash.py::TestGLM53FlashFP8::test_runtime # ------------- modules (non-MoE) --------------- - unittest/_torch/modules/test_fused_add_rms_norm_quant.py - unittest/_torch/modules/test_fused_activation_quant.py From 6bbd17c75a902c0b57db6e13609277f7d5a10d5c Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Wed, 23 Sep 2026 02:32:44 -0700 Subject: [PATCH 27/35] [None][doc] Document GLM-5.3-Flash long-input benchmark results Signed-off-by: Ruocheng Jia --- ...yment-guide-for-glm-5.3-flash-on-trtllm.md | 36 ++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md index 8b07ace3fdb0..5237107bd5c3 100644 --- a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md @@ -443,6 +443,10 @@ $$ ## Performance +### ISL 1024 / OSL 1024 + +This sweep was recorded before the small-batch MoE and native 16-head attention optimizations used in the long-input measurements below. + The chart compares TP4 / EP4 and attention DP4 / EP4, each with and without MTP3, on 4x B200 with FP8 weights and a BF16 KV cache. Measurements use the `benchmark_serving` client above, ISL 1024 / OSL 1024, random token IDs, seed 0, and greedy decoding. Each run sends `5 * concurrency` requests. Points combine all matching runs; faint dots show individual runs and bars show their minimum and maximum, not confidence intervals. To reproduce the curve, use the serving configurations above with `--max_batch_size 128`, `--max_seq_len 8192`, and `cuda_graph_config.max_batch_size: 128`. Keep CUDA graph padding, chunked prefill, and the overlap scheduler enabled; use a cache memory fraction of 0.5 and disable block reuse. Set `--max_num_tokens 16384` for TP4 / EP4 or `4096` for attention DP4 / EP4, with or without MTP3. @@ -451,4 +455,34 @@ The horizontal axis is `1000 / mean_tpot_ms`, excluding TTFT. The vertical axis ![GLM-5.3-Flash FP8 performance on 4x B200](../media/glm_5_3_flash_fp8_perf.png) -With TP4 / EP4, MTP3 improves single-user decode speed from approximately 153 to 372 tok/s/user. At concurrency 128, the four configurations deliver approximately 5.9K–6.3K output tok/s in aggregate. MTP acceptance and speedup depend on the workload; these measurements use random-token prompts. +In this sweep, TP4 / EP4 with MTP3 achieved approximately 372 tok/s/user versus 153 without MTP. At concurrency 128, the four configurations delivered approximately 5.9K–6.3K output tok/s in aggregate. MTP acceptance and speedup depend on the workload; these measurements use random-token prompts. + +### ISL 13312 / OSL 33 + +The following results were recorded on September 21, 2026, with the implementation committed as `26f2f5bc67`, which included a dedicated FP32 router kernel that has since been removed. These results apply to that revision and have not been revalidated on the current implementation. They use 4x B200, official FP8 weights, BF16 KV cache, and no MTP. Each row combines three runs of 64 requests with random token-ID prompts, seed 0, and greedy decoding. + +| Concurrency | Configuration | `max_num_tokens` | TTFT p50 (ms) | TPOT p50 (ms) | Aggregate output tok/s | +|---|---|---:|---:|---:|---:| +| 1 | TP4 / EP4 | 16384 | 307.38 | 5.57 | 67.91 | +| 8 | TP4 / EP4 | 16384 | 909.18 | 54.33 | 99.58 | +| 8 | Attention DP4 / EP4 | 4096 per rank | 1214.69 | 23.46 | 133.71 | + +At concurrency 1, decode speed is approximately 179.6 tok/s/user, calculated as `1000 / p50_tpot_ms` from the unrounded value and excluding TTFT. Aggregate output throughput includes both prefill and decode time. At concurrency 8, attention DP increases aggregate throughput by approximately 34% relative to TP, with higher TTFT. + +Use the serving configurations above without `speculative_config`, with `--max_batch_size 128`, `--max_seq_len 16384`, and `cuda_graph_config.max_batch_size: 128`. Keep CUDA graph padding, chunked prefill, and the overlap scheduler enabled; set the cache memory fraction to 0.5 and disable block reuse. Select the attention mode and token budget from the table. + +After a separate warmup, run this benchmark three times for each configuration, setting concurrency to 1 or 8 and saving each run to a separate result directory: + +```bash +python -m tensorrt_llm.serve.scripts.benchmark_serving \ + --model zai-org/GLM-5.3-Flash --backend openai \ + --host 127.0.0.1 --port 8000 \ + --dataset-name random --random-input-len 13312 --random-output-len 33 \ + --random-prefix-len 0 --random-ids \ + --num-prompts 64 --max-concurrency 1 \ + --ignore-eos --tokenize-on-client --seed 0 --temperature 0 \ + --percentile-metrics ttft,tpot,itl,e2el \ + --save-result --save-detailed --result-dir ./glm53_c1_run1 +``` + +Compute TTFT and TPOT p50 over all 192 request samples. Aggregate output throughput is total generated tokens divided by the sum of the three run durations. From 55ce5099a0004ef5aa8596b46fbaec3430f20bb9 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Wed, 23 Sep 2026 04:57:46 -0700 Subject: [PATCH 28/35] [None][doc] Restore GLM-5.3-Flash performance guide Signed-off-by: Ruocheng Jia --- ...yment-guide-for-glm-5.3-flash-on-trtllm.md | 36 +------------------ 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md index 5237107bd5c3..8b07ace3fdb0 100644 --- a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md @@ -443,10 +443,6 @@ $$ ## Performance -### ISL 1024 / OSL 1024 - -This sweep was recorded before the small-batch MoE and native 16-head attention optimizations used in the long-input measurements below. - The chart compares TP4 / EP4 and attention DP4 / EP4, each with and without MTP3, on 4x B200 with FP8 weights and a BF16 KV cache. Measurements use the `benchmark_serving` client above, ISL 1024 / OSL 1024, random token IDs, seed 0, and greedy decoding. Each run sends `5 * concurrency` requests. Points combine all matching runs; faint dots show individual runs and bars show their minimum and maximum, not confidence intervals. To reproduce the curve, use the serving configurations above with `--max_batch_size 128`, `--max_seq_len 8192`, and `cuda_graph_config.max_batch_size: 128`. Keep CUDA graph padding, chunked prefill, and the overlap scheduler enabled; use a cache memory fraction of 0.5 and disable block reuse. Set `--max_num_tokens 16384` for TP4 / EP4 or `4096` for attention DP4 / EP4, with or without MTP3. @@ -455,34 +451,4 @@ The horizontal axis is `1000 / mean_tpot_ms`, excluding TTFT. The vertical axis ![GLM-5.3-Flash FP8 performance on 4x B200](../media/glm_5_3_flash_fp8_perf.png) -In this sweep, TP4 / EP4 with MTP3 achieved approximately 372 tok/s/user versus 153 without MTP. At concurrency 128, the four configurations delivered approximately 5.9K–6.3K output tok/s in aggregate. MTP acceptance and speedup depend on the workload; these measurements use random-token prompts. - -### ISL 13312 / OSL 33 - -The following results were recorded on September 21, 2026, with the implementation committed as `26f2f5bc67`, which included a dedicated FP32 router kernel that has since been removed. These results apply to that revision and have not been revalidated on the current implementation. They use 4x B200, official FP8 weights, BF16 KV cache, and no MTP. Each row combines three runs of 64 requests with random token-ID prompts, seed 0, and greedy decoding. - -| Concurrency | Configuration | `max_num_tokens` | TTFT p50 (ms) | TPOT p50 (ms) | Aggregate output tok/s | -|---|---|---:|---:|---:|---:| -| 1 | TP4 / EP4 | 16384 | 307.38 | 5.57 | 67.91 | -| 8 | TP4 / EP4 | 16384 | 909.18 | 54.33 | 99.58 | -| 8 | Attention DP4 / EP4 | 4096 per rank | 1214.69 | 23.46 | 133.71 | - -At concurrency 1, decode speed is approximately 179.6 tok/s/user, calculated as `1000 / p50_tpot_ms` from the unrounded value and excluding TTFT. Aggregate output throughput includes both prefill and decode time. At concurrency 8, attention DP increases aggregate throughput by approximately 34% relative to TP, with higher TTFT. - -Use the serving configurations above without `speculative_config`, with `--max_batch_size 128`, `--max_seq_len 16384`, and `cuda_graph_config.max_batch_size: 128`. Keep CUDA graph padding, chunked prefill, and the overlap scheduler enabled; set the cache memory fraction to 0.5 and disable block reuse. Select the attention mode and token budget from the table. - -After a separate warmup, run this benchmark three times for each configuration, setting concurrency to 1 or 8 and saving each run to a separate result directory: - -```bash -python -m tensorrt_llm.serve.scripts.benchmark_serving \ - --model zai-org/GLM-5.3-Flash --backend openai \ - --host 127.0.0.1 --port 8000 \ - --dataset-name random --random-input-len 13312 --random-output-len 33 \ - --random-prefix-len 0 --random-ids \ - --num-prompts 64 --max-concurrency 1 \ - --ignore-eos --tokenize-on-client --seed 0 --temperature 0 \ - --percentile-metrics ttft,tpot,itl,e2el \ - --save-result --save-detailed --result-dir ./glm53_c1_run1 -``` - -Compute TTFT and TPOT p50 over all 192 request samples. Aggregate output throughput is total generated tokens divided by the sum of the three run durations. +With TP4 / EP4, MTP3 improves single-user decode speed from approximately 153 to 372 tok/s/user. At concurrency 128, the four configurations deliver approximately 5.9K–6.3K output tok/s in aggregate. MTP acceptance and speedup depend on the workload; these measurements use random-token prompts. From c1140b4f140abd2c4953da15a395b7f532d26ed6 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Wed, 23 Sep 2026 19:35:59 -0700 Subject: [PATCH 29/35] [None][fix] Account for GLM indexer cache memory Signed-off-by: Ruocheng Jia --- .../sparse/glm_kpool/cache_manager.py | 100 ++++++++++- .../sparse/glm_kpool/test_glm_kpool.py | 158 ++++++++++++++++++ 2 files changed, 254 insertions(+), 4 deletions(-) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py index e4aa9d419147..cefba7c98115 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py @@ -20,9 +20,17 @@ import torch +from tensorrt_llm._torch.model_config import ModelConfig +from tensorrt_llm._torch.pyexecutor.config_utils import ( + extract_mamba_kv_cache_params, + unwrap_glm5_next_text_config, +) from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import Role from tensorrt_llm._torch.pyexecutor.kv_cache.mamba_cache_manager import MambaHybridCacheManagerV2 -from tensorrt_llm.runtime.kv_cache_manager_v2 import BufferConfig +from tensorrt_llm._torch.pyexecutor.resource_manager import get_pp_layers +from tensorrt_llm.llmapi.llm_args import KvCacheConfig +from tensorrt_llm.mapping import Mapping +from tensorrt_llm.runtime.kv_cache_manager_v2 import BufferConfig, DataRole class Glm5NextCacheManager(MambaHybridCacheManagerV2): @@ -50,14 +58,98 @@ def __init__( def _extra_buffers_per_layer(self, *, tokens_per_block: int) -> dict[int, list[BufferConfig]]: """One ``Role.INDEX_KEY`` buffer per sparse layer, keyed by local id.""" - elem_bytes = torch.tensor([], dtype=torch.bfloat16).element_size() - size_per_block = self.index_state_dim * elem_bytes * tokens_per_block return { - self.layer_offsets[layer_id]: [BufferConfig(role=Role.INDEX_KEY, size=size_per_block)] + self.layer_offsets[layer_id]: [ + BufferConfig( + role=Role.INDEX_KEY, + size=self.get_layer_bytes_per_token( + self.layer_offsets[layer_id], Role.INDEX_KEY + ) + * tokens_per_block, + ) + ] for layer_id in self.sparse_layer_ids if layer_id in self.layer_offsets } + def get_layer_bytes_per_token(self, local_layer_idx: int, data_role: DataRole) -> int: + index_bytes = ( + self.index_state_dim * torch.bfloat16.itemsize + if self.pp_layers[local_layer_idx] in self.sparse_layer_ids + else 0 + ) + if data_role == Role.INDEX_KEY: + return index_bytes + cache_bytes = super().get_layer_bytes_per_token(local_layer_idx, data_role) + return cache_bytes + index_bytes if data_role == Role.ALL else cache_bytes + + def _attention_cache_bytes_per_token(self) -> int: + return sum( + self.get_layer_bytes_per_token(local_layer_idx, Role.ALL) + for local_layer_idx in range(self.num_local_layers) + ) + + @staticmethod + def get_cache_size_per_token( + model_config: ModelConfig, + mapping: Mapping, + *, + max_batch_size: int, + kv_cache_config: KvCacheConfig, + tokens_per_block: int = 32, + max_seq_len: int | None = None, + **kwargs, + ) -> tuple[int, int]: + slope, fixed_cost = MambaHybridCacheManagerV2.get_cache_size_per_token( + model_config, + mapping, + max_batch_size=max_batch_size, + kv_cache_config=kv_cache_config, + tokens_per_block=tokens_per_block, + max_seq_len=max_seq_len, + **kwargs, + ) + spec_config = kwargs.get("spec_config") + params = extract_mamba_kv_cache_params( + model_config.pretrained_config, + spec_config=spec_config, + quant_config=model_config.quant_config, + ) + kda_mask, attention_mask = params.get_layer_masks( + is_draft=kwargs.get("is_draft", False), + use_separate_draft_kv_cache=kwargs.get("use_separate_draft_kv_cache", False), + ) + layer_mask = [kda or attention for kda, attention in zip(kda_mask, attention_mask)] + local_layers, _ = get_pp_layers( + sum(layer_mask), mapping, spec_config=spec_config, layer_mask=layer_mask + ) + local_attention_layers = sum(attention_mask[layer] for layer in local_layers) + config = unwrap_glm5_next_text_config(model_config.pretrained_config) + # All GLM full-attention layers, including MTP layers, use the sparse indexer. + # Indexer pages remain BF16 even when latent KV is quantized. + index_bytes_per_token = ( + local_attention_layers * 3 * config.index_head_dim * torch.bfloat16.itemsize + ) + state_config = kv_cache_config.mamba_state_config + seq_limit = max_seq_len if max_seq_len is not None else float("inf") + has_snapshots = kv_cache_config.enable_block_reuse and ( + 0 < state_config.periodic_snapshot_interval <= seq_limit + or any( + offset <= seq_limit + for offset in state_config.additional_snapshot_offsets_from_start + ) + or any( + offset < seq_limit for offset in state_config.additional_snapshot_offsets_from_end + ) + ) + # V2 reserves one partial attention page per resident lineage when snapshots + # are reachable. Include the indexer section of those retained pages. + if has_snapshots: + fixed_cost += ( + max_batch_size * mapping.pp_size * tokens_per_block * index_bytes_per_token + ) + return slope + index_bytes_per_token, fixed_cost + def get_index_state_buffer(self, layer_idx: int) -> torch.Tensor | None: """Paged indexer state for ``layer_idx``, NHD-shaped.""" return self.get_index_k_buffer( diff --git a/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py b/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py index 526834f7fd65..e989db333636 100644 --- a/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py +++ b/tests/unittest/_torch/attention/sparse/glm_kpool/test_glm_kpool.py @@ -14,6 +14,7 @@ # limitations under the License. """CPU tests for GLM sparse attention metadata and cache layouts.""" +from copy import deepcopy from types import SimpleNamespace from unittest.mock import Mock, patch @@ -38,6 +39,163 @@ pytestmark = pytest.mark.cpu_only +@pytest.mark.parametrize("fp8", [False, True], ids=["bf16-kv", "fp8-kv"]) +@pytest.mark.parametrize( + "snapshot_options,retains_page", + [ + ({}, False), + ({"periodic_snapshot_interval": 32}, True), + ({"periodic_snapshot_interval": 48}, True), + ({"periodic_snapshot_interval": 1024}, False), + ({"additional_snapshot_offsets_from_start": [512]}, True), + ({"additional_snapshot_offsets_from_start": [513]}, False), + ({"additional_snapshot_offsets_from_end": [511]}, True), + ({"additional_snapshot_offsets_from_end": [512]}, False), + ], + ids=[ + "no-snapshots", + "aligned-periodic", + "unaligned-periodic", + "unreachable-periodic", + "start-boundary", + "unreachable-start", + "from-end", + "unreachable-end", + ], +) +@pytest.mark.parametrize( + "layout,expected_sparse_layers", + [("tp", 1), ("adp", 1), ("mtp", 2), ("pp-linear", 0), ("pp-mtp", 2), ("draft-only", 1)], +) +def test_indexer_static_cache_cost( + fp8, snapshot_options, retains_page, layout, expected_sparse_layers +): + from tensorrt_llm._torch.attention.backends.sparse.glm_kpool.cache_manager import ( + Glm5NextCacheManager, + ) + from tensorrt_llm._torch.configs.glm5_next import Glm5NextTextConfig + from tensorrt_llm._torch.model_config import ModelConfig + from tensorrt_llm._torch.pyexecutor.kv_cache.mamba_cache_manager import ( + MambaHybridCacheManagerV2, + ) + from tensorrt_llm.llmapi import KvCacheConfig, MambaStateConfig, MTPDecodingConfig + from tensorrt_llm.mapping import Mapping + from tensorrt_llm.models.modeling_utils import QuantConfig + + config = Glm5NextTextConfig( + num_hidden_layers=2, + layer_types=["linear_attention", "deepseek_sparse_attention"], + kv_lora_rank=512, + index_head_dim=128, + dtype="bfloat16", + linear_attn_config={"num_heads": 4, "head_dim": 8, "short_conv_kernel_size": 4}, + ) + model_config = ModelConfig( + pretrained_config=config, + quant_config=QuantConfig(kv_cache_quant_algo="FP8" if fp8 else None), + ) + pp_size = 2 if layout.startswith("pp-") else 1 + mapping = Mapping( + world_size=4 * pp_size, + tp_size=4, + pp_size=pp_size, + pp_partition=[1, 1] if pp_size == 2 else None, + rank=4 if layout == "pp-mtp" else 0, + enable_attention_dp=layout == "adp", + ) + kwargs = dict( + max_batch_size=4, + tokens_per_block=32, + max_seq_len=512, + kv_cache_config=KvCacheConfig( + enable_block_reuse=bool(snapshot_options), + mamba_state_config=MambaStateConfig(**snapshot_options), + ), + spec_config=MTPDecodingConfig(max_draft_len=3) + if layout in ("mtp", "pp-mtp", "draft-only") + else None, + is_draft=layout == "draft-only", + ) + base_slope, base_fixed = MambaHybridCacheManagerV2.get_cache_size_per_token( + model_config, mapping, **kwargs + ) + slope, fixed = Glm5NextCacheManager.get_cache_size_per_token(model_config, mapping, **kwargs) + # The three 128-wide BF16 indexer sections are replicated on every TP/ADP rank. + extra_per_token = expected_sparse_layers * 3 * 128 * 2 + assert slope - base_slope == extra_per_token + retained_pages = 4 * pp_size if retains_page else 0 + assert fixed - base_fixed == retained_pages * 32 * extra_per_token + # An equivalent number of latent bytes must get the same token and snapshot + # budget from the base estimator, independently of where the bytes are stored. + reference_config = deepcopy(model_config) + reference_config.pretrained_config.kv_lora_rank += 768 if fp8 else 384 + assert (slope, fixed) == MambaHybridCacheManagerV2.get_cache_size_per_token( + reference_config, mapping, **kwargs + ) + + +@pytest.mark.parametrize("fp8", [False, True], ids=["bf16-kv", "fp8-kv"]) +@pytest.mark.parametrize("snapshot_interval", [0, 32]) +def test_indexer_runtime_cache_cost_matches_registered_buffers(fp8, snapshot_interval): + from tensorrt_llm._torch.attention.backends.sparse.glm_kpool.cache_manager import ( + Glm5NextCacheManager, + ) + from tensorrt_llm._torch.pyexecutor.kv_cache.kv_cache_manager_v2 import Role + from tensorrt_llm._torch.pyexecutor.resource_manager import CacheTypeCpp, DataType + from tensorrt_llm.llmapi import KvCacheConfig, MambaStateConfig + from tensorrt_llm.mapping import Mapping + + manager = object.__new__(Glm5NextCacheManager) + manager.mapping = Mapping(world_size=1, tp_size=1) + manager.pp_layers = [0, 1] + manager.layer_offsets = {0: 0, 1: 1} + manager.sparse_layer_ids = [1, 3] # Layer 3 is on another PP rank. + manager.index_state_dim = 384 + manager.num_local_layers = 2 + manager.local_num_mamba_layers = 1 + manager.num_kv_heads_per_layer = [0, 1] + manager.head_dim_per_layer = [512, 512] + manager.kv_cache_type = CacheTypeCpp.SELFKONLY + manager.kv_factor = 1 + manager.dtype = DataType.FP8 if fp8 else DataType.BF16 + manager.tokens_per_block = 32 + manager.max_batch_size = 4 + manager.max_num_tokens = 16 + manager.max_seq_len = 512 + manager.max_attention_window_vec = [None, None] + manager.enable_swa_scratch_reuse = False + manager._generation_kv_capacity_headroom = 0 + manager._has_cp_helix = False + manager._num_reserved_dummy_slots = 1 + manager._ple_layer_ids = [] + manager.ssm_bytes, manager.conv_bytes = 256, 144 + manager.kv_cache_config = KvCacheConfig( + enable_block_reuse=bool(snapshot_interval), + mamba_state_config=MambaStateConfig(periodic_snapshot_interval=snapshot_interval), + ) + + buffers = manager._extra_buffers_per_layer(tokens_per_block=32) + assert set(buffers) == {1} + assert buffers[1][0].role == Role.INDEX_KEY + assert buffers[1][0].size == 768 * 32 + latent_bytes = 512 if fp8 else 1024 + assert manager.get_layer_bytes_per_token(0, Role.ALL) == 0 + assert manager.get_layer_bytes_per_token(0, Role.INDEX_KEY) == 0 + assert manager.get_layer_bytes_per_token(1, Role.KEY) == latent_bytes + assert manager.get_layer_bytes_per_token(1, Role.INDEX_KEY) == 768 + assert manager.get_layer_bytes_per_token(1, Role.ALL) == latent_bytes + 768 + assert manager._attention_cache_bytes_per_token() == latent_bytes + 768 + snapshot_bytes_per_token = 400 // 32 if snapshot_interval else 0 + assert manager.get_cache_bytes_per_token() == latent_bytes + 768 + snapshot_bytes_per_token + + tokens = 128 + state_slots = 5 + (tokens // 32 if snapshot_interval else 0) + retained_page_tokens = 4 * 32 if snapshot_interval else 0 + expected_quota = (tokens + retained_page_tokens) * (latent_bytes + 768) + state_slots * 400 + assert manager._get_quota_from_max_tokens(tokens) == expected_quota + assert manager._get_max_tokens_from_quota(expected_quota) == tokens + + def test_nope_mla_geometry_uses_shared_factory(monkeypatch): from tensorrt_llm._torch.attention.backends.trtllm import TrtllmAttention from tensorrt_llm._torch.attention.backends.utils import create_attention From de9f05c03a284ff269379abd5bf4c7484f02a24f Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Fri, 25 Sep 2026 10:43:57 -0700 Subject: [PATCH 30/35] [None][refactor] Address GLM-5.3-Flash review follow-ups Let Glm5NextCacheManager derive its indexer layers from the attention layer mask it already receives, so the shared KV cache factory passes only index_state_dim, and compute the [k | gate | pool key] row width through one glm_kpool_cache_row_dim helper shared by the factory, the cache estimator and the model. Rename the factory's is_glm flag to is_glm5_next, drop its redundant replay-manager check, and restore the Kimi K3 hybrid cache comments in the shared factory branch. Drop a greedy stop-word test that duplicates the one already on main, skip the MMMU test when the native glm5_next processor is unavailable, point the deployment guide at 1.3.0rc29, scope the transformers 5.17.0 requirement to image and video inputs, and align the GLM file headers and footnote numbering with the rest of the tree. Signed-off-by: Ruocheng Jia --- ...yment-guide-for-glm-5.3-flash-on-trtllm.md | 6 +-- docs/source/models/supported-models.md | 4 +- .../backends/sparse/glm_kpool/__init__.py | 3 +- .../sparse/glm_kpool/cache_manager.py | 29 +++++++++- .../backends/sparse/glm_kpool/params.py | 11 +++- .../checkpoints/hf/glm5_next_weight_mapper.py | 2 +- .../_torch/models/modeling_glm5_next.py | 5 +- .../models/modeling_glm5_next_vision.py | 14 ++++- tensorrt_llm/_torch/pyexecutor/_util.py | 53 ++++++++++++------- .../_torch/pyexecutor/config_utils.py | 2 +- .../defs/accuracy/test_glm53_flash.py | 1 + .../_torch/sampler/test_torch_sampler.py | 32 ----------- 12 files changed, 97 insertions(+), 65 deletions(-) diff --git a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md index 8b07ace3fdb0..98d1f5cc4403 100644 --- a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md @@ -34,8 +34,8 @@ See the [Model-Feature Support Matrix](../models/supported-models.md#model-featu * OS: Linux * Drivers: CUDA Driver 575 or later * Docker with NVIDIA Container Toolkit installed -* Minimum TensorRT LLM version: 1.3.0rc26 -* Install `transformers==5.17.0` in the GLM deployment environment for model configuration and image/video processing: +* Minimum TensorRT LLM version: 1.3.0rc29 +* Text-only serving works with the Transformers version installed with TensorRT LLM. Image and video inputs require the native `Glm5NextProcessor` from `transformers==5.17.0`; install it in the GLM deployment environment: ```bash pip install "transformers==5.17.0" @@ -76,7 +76,7 @@ docker run --rm -it \ -p 8000:8000 \ -v /path/to/your/models:/models \ --name tensorrt_llm \ - nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc26 \ + nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc29 \ /bin/bash ``` diff --git a/docs/source/models/supported-models.md b/docs/source/models/supported-models.md index 492a33d7d2e4..025de6db73fb 100644 --- a/docs/source/models/supported-models.md +++ b/docs/source/models/supported-models.md @@ -21,7 +21,7 @@ The following is a table of supported models for the PyTorch backend: | `Gemma4AssistantForCausalLM` | Gemma 4 MTP assistant | `google/gemma-4-E2B-it-assistant`, `google/gemma-4-E4B-it-assistant`, `google/gemma-4-26B-A4B-it-assistant`, `google/gemma-4-31B-it-assistant` | | `Glm4MoeForCausalLM` | GLM-4.5, GLM-4.6, GLM-4.7 | `THUDM/GLM-4-100B-A10B` | | `GlmMoeDsaForCausalLM` | GLM-5, GLM-5.2, GLM-5.3 | `zai-org/GLM-5`, `zai-org/GLM-5.2`, `zai-org/GLM-5.3` | -| `Glm5NextForConditionalGeneration` [^21] | GLM-5.3-Flash | `zai-org/GLM-5.3-Flash` | +| `Glm5NextForConditionalGeneration` [^20] | GLM-5.3-Flash | `zai-org/GLM-5.3-Flash` | | `GptOssForCausalLM` | GPT-OSS | `openai/gpt-oss-20b`, `openai/gpt-oss-120b` | | `KimiK25ForConditionalGeneration` | Kimi-K2.5 | `moonshotai/Kimi-K2.5` | | `KimiK3ForConditionalGeneration` [^15]| Kimi-K3 | `moonshotai/Kimi-K3` | @@ -96,7 +96,7 @@ statuses for the same architecture in the two matrices. [^17]: Kimi K3 has no MTP or EAGLE-3 head, and its DSpark checkpoints are not compatible with plain `DFlash`. [^18]: NGram and standalone Suffix Automaton (SA) use model-free drafting on the PyTorch backend, so they are not listed in individual entries. This does not imply universal end-to-end support: compatibility depends on each model's multi-token verification and cache-management paths and may be untested or explicitly restricted. [^19]: KV cache reuse for hybrid recurrent-attention models requires an explicit recurrent-state snapshot policy, such as `kv_cache_config.mamba_state_config.periodic_snapshot_interval`; the model default disables reuse when no snapshot policy is configured. -[^21]: Supports text, image, and video inputs, MTP (including with attention data parallelism), and FP8 KV cache. Requires `transformers==5.17.0`. Beam search is not supported. See the [GLM-5.3-Flash deployment guide](../deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md) for setup and feature-specific requirements. +[^20]: Supports text, image, and video inputs, MTP (including with attention data parallelism), and FP8 KV cache. Image and video inputs require `transformers==5.17.0`. Beam search is not supported. See the [GLM-5.3-Flash deployment guide](../deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md) for setup and feature-specific requirements. # Encoder-Decoder Feature Support Matrix (PyTorch Backend) diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/__init__.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/__init__.py index c8d4638792ee..439bd989e3b1 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/__init__.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/__init__.py @@ -17,7 +17,7 @@ from .backend import INDEX_SENTINEL, GlmKpoolSparseAttention, latent_pool_rows, paged_slot_indices from .cache_manager import Glm5NextCacheManager from .metadata import Glm5NextMamba2Metadata -from .params import GlmKpoolBackendForwardArgs, GlmKpoolSparseParams +from .params import GlmKpoolBackendForwardArgs, GlmKpoolSparseParams, glm_kpool_cache_row_dim __all__ = [ "INDEX_SENTINEL", @@ -26,6 +26,7 @@ "GlmKpoolBackendForwardArgs", "GlmKpoolSparseAttention", "GlmKpoolSparseParams", + "glm_kpool_cache_row_dim", "latent_pool_rows", "paged_slot_indices", ] diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py index cefba7c98115..85d5596065f1 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/cache_manager.py @@ -32,6 +32,8 @@ from tensorrt_llm.mapping import Mapping from tensorrt_llm.runtime.kv_cache_manager_v2 import BufferConfig, DataRole +from .params import glm_kpool_cache_row_dim + class Glm5NextCacheManager(MambaHybridCacheManagerV2): """Manage KDA state, latent KV and indexer buffers in one V2 lifecycle. @@ -43,10 +45,31 @@ class Glm5NextCacheManager(MambaHybridCacheManagerV2): """ def __init__( - self, *args, sparse_layer_ids: Sequence[int] = (), index_state_dim: int = 0, **kwargs + self, + *args, + sparse_layer_ids: Sequence[int] | None = None, + index_state_dim: int = 0, + **kwargs, ) -> None: + """Build the hybrid manager with one indexer buffer per sparse layer. + + Args: + sparse_layer_ids: Global layer ids that carry an indexer buffer. + Defaults to every attention layer in ``layer_mask``: each GLM + attention layer, including appended MTP layers, is sparse. + index_state_dim: BF16 width of one indexer cache row, see + :func:`glm_kpool_cache_row_dim`. + """ # Set before super().__init__: the base _build_base_config calls # _extra_buffers_per_layer, which reads both of these. + if sparse_layer_ids is None: + layer_mask = kwargs.get("layer_mask") + if layer_mask is None: + raise ValueError( + "Glm5NextCacheManager needs layer_mask or sparse_layer_ids " + "to place the indexer buffers" + ) + sparse_layer_ids = [i for i, is_attention in enumerate(layer_mask) if is_attention] self.sparse_layer_ids = sorted(int(i) for i in sparse_layer_ids) self.index_state_dim = int(index_state_dim) if self.sparse_layer_ids and self.index_state_dim <= 0: @@ -128,7 +151,9 @@ def get_cache_size_per_token( # All GLM full-attention layers, including MTP layers, use the sparse indexer. # Indexer pages remain BF16 even when latent KV is quantized. index_bytes_per_token = ( - local_attention_layers * 3 * config.index_head_dim * torch.bfloat16.itemsize + local_attention_layers + * glm_kpool_cache_row_dim(int(config.index_head_dim)) + * torch.bfloat16.itemsize ) state_config = kv_cache_config.mamba_state_config seq_limit = max_seq_len if max_seq_len is not None else float("inf") diff --git a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/params.py b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/params.py index 17a3d7265f07..b43a9881d4a1 100644 --- a/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/params.py +++ b/tensorrt_llm/_torch/attention/backends/sparse/glm_kpool/params.py @@ -30,6 +30,15 @@ INDEX_SENTINEL = -1 +def glm_kpool_cache_row_dim(index_head_dim: int) -> int: + """Width of one indexer cache row, ``[k | gate | pool key]``. + + Each field is ``index_head_dim`` wide. The cache manager sizes the paged + indexer buffers with this width. + """ + return 3 * index_head_dim + + @dataclass(frozen=True) class GlmKpoolSparseParams(SparseParams): """Lowered runtime parameters for the GLM k-pool sparse-MLA backend.""" @@ -72,7 +81,7 @@ def cache_row_dim(self) -> int: it is maintained incrementally by :meth:`GlmKpoolSparseAttention. update_pool_keys` so decode never rebuilds pools from scratch. """ - return 3 * self.index_head_dim + return glm_kpool_cache_row_dim(self.index_head_dim) @property def select_k(self) -> int: diff --git a/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py b/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py index ca8297d61166..7293db1d0000 100644 --- a/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py +++ b/tensorrt_llm/_torch/models/checkpoints/hf/glm5_next_weight_mapper.py @@ -5,7 +5,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next.py b/tensorrt_llm/_torch/models/modeling_glm5_next.py index 6542f2b9b653..fed15d3ca435 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next.py @@ -5,7 +5,7 @@ # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # -# http://www.apache.org/licenses/LICENSE-2.0 +# http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, @@ -43,6 +43,7 @@ Glm5NextMamba2Metadata, GlmKpoolBackendForwardArgs, GlmKpoolSparseParams, + glm_kpool_cache_row_dim, ) from ..attention.backends.utils import create_attention from ..distributed import AllReduceStrategy @@ -962,7 +963,7 @@ def __init__( @property def cache_state_dim(self) -> int: """Width of a cached [key | compression gate | pooled key] row.""" - return 3 * self.head_dim + return glm_kpool_cache_row_dim(self.head_dim) def project_state(self, hidden_states: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: """``(packed [k(head_dim) | gate(head_dim)], head weights [n_heads])`` diff --git a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py index 98657afc3cb2..6bb3500c0dde 100644 --- a/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py +++ b/tensorrt_llm/_torch/models/modeling_glm5_next_vision.py @@ -1,5 +1,17 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 -# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """GLM-5.3-Flash vision encoder and conditional-generation wrapper. Images and video frame pairs share a BF16 tower: patch embedding, pre-norm diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index c23fd91c5688..6a38a4e904ae 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -3003,19 +3003,26 @@ def _create_kv_cache_manager( manager_extra_kwargs["is_disagg"] = is_disagg if config_utils.is_glm5_next(config) or is_kimi_linear(config): - is_glm = config_utils.is_glm5_next(config) + # Kimi K3 hybrid: KDA (Kimi Delta Attention) recurrent/conv states on + # the mamba side of the hybrid manager, absorbed-MQA MLA latent cache + # (num_kv_heads=1, head_dim = kv_lora_rank + qk_rope_head_dim, + # SELFKONLY) on the paged-KV side. Must come before the is_mla(...) + # route: the kimi_linear config carries MLA fields, but only 24 of + # its 93 layers are MLA. glm5_next has the same layout (11 of its 45 + # layers are sparse MLA) plus paged indexer state; both families use + # sequential verify when replay is unavailable. + is_glm5_next = config_utils.is_glm5_next(config) text_config = config_utils.unwrap_glm5_next_text_config( - config) if is_glm else config - # Both families combine KDA state with latent MLA pages. GLM adds - # paged indexer state; both use sequential verify when replay is unavailable. + config) if is_glm5_next else config if max_beam_width > 1: raise ValueError( - "glm5_next + beam search is not supported yet." if is_glm else + "glm5_next + beam search is not supported yet." + if is_glm5_next else "MambaHybridCacheManager + beam search is not supported yet.") if not estimating_kv_cache and kv_connector_manager is not None: raise NotImplementedError( "Connector manager is not supported for glm5_next." - if is_glm else + if is_glm5_next else "Connector manager is not supported for MambaHybridCacheManager." ) mamba_params = extract_mamba_kv_cache_params( @@ -3032,26 +3039,34 @@ def _create_kv_cache_manager( )) num_mamba_layers = (0 if is_draft and mamba_params.num_draft_layers > 0 else mamba_params.num_mamba_layers) - # Pass full KDA head counts: the manager applies attention-TP sharding. + # KDA state sharding follows the attention-family TP + # semantics (Qwen3-Next pattern): replicated under attention-DP, + # head-sharded across tp_size otherwise. That is exactly the cache + # manager's own internal gate (`tp_size = 1 if enable_attention_dp + # else tp_size`, then num_heads / n_groups / conv_dim divide by + # it), so the params pass through unscaled. + # KDA fused multi-token verify (trtllm::kda_mtp_decode): when the + # kernel can run here, allocate the per-slot replay caches instead + # of the legacy per-step intermediate verification buffers. The + # kernel replays accepted drafts from these caches and commits + # states in place, replacing the intermediate-buffer + promotion + # flow for KDA layers. kda_extra_kwargs = {} kda_replay_manager_types = (MixedMambaHybridCacheManager, MambaHybridCacheManagerV2) - if (spec_config is not None and - (is_glm - or issubclass(kv_cache_manager_cls, kda_replay_manager_types))): + if (spec_config is not None + and issubclass(kv_cache_manager_cls, kda_replay_manager_types)): from ..modules.kimi_kda._kda_kernels import \ is_kda_mtp_verify_available if is_kda_mtp_verify_available(): kda_extra_kwargs["kda_replay_num_spec"] = ( spec_config.tokens_per_gen_step - 1) - if is_glm: - kda_extra_kwargs.update( - sparse_layer_ids=[ - i for i, sparse in enumerate(full_attention_layer_mask) - if sparse - ], - index_state_dim=3 * int(text_config.index_head_dim), - ) + if is_glm5_next: + # The manager places an indexer buffer on every attention layer. + from ..attention.backends.sparse.glm_kpool import \ + glm_kpool_cache_row_dim + kda_extra_kwargs["index_state_dim"] = glm_kpool_cache_row_dim( + int(text_config.index_head_dim)) # KDA's conv state is a [Q | K | V] concatenation whose three sections # have identical width, i.e. the qwen3_next section layout. kda_extra_kwargs.update( @@ -3075,7 +3090,7 @@ def _create_kv_cache_manager( num_kv_heads=1, head_dim=(int(text_config.kv_lora_rank) + int(getattr(text_config, "qk_rope_head_dim", 0) or 0) - if is_glm else config.kv_lora_rank + + if is_glm5_next else config.kv_lora_rank + config.qk_rope_head_dim), tokens_per_block=tokens_per_block, max_seq_len=max_seq_len, diff --git a/tensorrt_llm/_torch/pyexecutor/config_utils.py b/tensorrt_llm/_torch/pyexecutor/config_utils.py index d0ad17384404..ce1d56453ea2 100644 --- a/tensorrt_llm/_torch/pyexecutor/config_utils.py +++ b/tensorrt_llm/_torch/pyexecutor/config_utils.py @@ -765,7 +765,7 @@ def extract_mamba_kv_cache_params( config, torch.bfloat16) validate_kimi_kda_state_dtype(config, mamba_ssm_cache_dtype) if is_glm5_next(config) and mamba_ssm_cache_dtype != torch.float32: - logger.info(f"GLM KDA: overriding mamba_ssm_cache_dtype " + logger.info(f"glm5_next KDA: overriding mamba_ssm_cache_dtype " f"{mamba_ssm_cache_dtype} -> torch.float32") mamba_ssm_cache_dtype = torch.float32 diff --git a/tests/integration/defs/accuracy/test_glm53_flash.py b/tests/integration/defs/accuracy/test_glm53_flash.py index d2c99fecd107..e7acda45b827 100644 --- a/tests/integration/defs/accuracy/test_glm53_flash.py +++ b/tests/integration/defs/accuracy/test_glm53_flash.py @@ -159,6 +159,7 @@ def _assert_kv_cache_reuse(llm: LLM) -> None: @pytest.mark.timeout(7200) @parametrize_with_ids("tp_size,ep_size", [(4, 4)]) def test_mmmu(self, tp_size, ep_size): + pytest.importorskip("transformers.models.glm5_next.processing_glm5_next") kwargs = self._llm_kwargs(tp_size, ep_size) kwargs["disable_mm_encoder"] = False # MMMU prompts fit in 8K (MAX_INPUT_LEN); a smaller token budget and diff --git a/tests/unittest/_torch/sampler/test_torch_sampler.py b/tests/unittest/_torch/sampler/test_torch_sampler.py index 62c73ab6699e..afa3e8e8359a 100644 --- a/tests/unittest/_torch/sampler/test_torch_sampler.py +++ b/tests/unittest/_torch/sampler/test_torch_sampler.py @@ -928,38 +928,6 @@ class TestFinishReasons: END_ID = FinishReason.END_ID LENGTH = FinishReason.LENGTH - @pytest.mark.parametrize("new_token", [5, 7, 9, 11]) - def test_single_step_greedy_checks_all_stop_tokens(self, new_token): - sampler = object.__new__(TorchSampler) - sampler.max_seq_len = 20 - sampler._track_pending_steps = False - request = LlmRequest( - request_id=0, - seq_slot=0, - input_tokens=[2, 0], - max_new_tokens=10, - end_id=2, - stop_words_list=[[5], [7], [9]], - sampling_config=SamplingConfig(), - is_streaming=False, - ) - state = SampleStateTorch( - requests=[request], - device=None, - host=SampleStateTensorsHostTorch( - new_tokens=torch.tensor([new_token], dtype=torch.int32), - finish_reasons=None, - first_finish_reasons=None, - single_step_greedy=True, - ), - ) - - sampler.update_requests(state) - - assert request.is_finished == (new_token in (5, 7, 9)) - assert not request.is_finished_due_to_length - assert request.get_tokens(0) == [2, 0, new_token] - def test_single_step_greedy_updates_finish_reasons_and_filters_completed_requests(self): sampler = object.__new__(TorchSampler) sampler.max_seq_len = 20 From f6a796e38a9641e84165d1154f535c0207396356 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Fri, 25 Sep 2026 10:52:20 -0700 Subject: [PATCH 31/35] [None][chore] Trim GLM KV cache manager routing comments Keep only the reason glm5_next rejects non-V2 managers and the Python NIXL transceiver requirement; the manager's buffer layout is already documented on Glm5NextCacheManager. Signed-off-by: Ruocheng Jia --- tensorrt_llm/_torch/pyexecutor/_util.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/_util.py b/tensorrt_llm/_torch/pyexecutor/_util.py index 6a38a4e904ae..4a1ac6a0ffc5 100644 --- a/tensorrt_llm/_torch/pyexecutor/_util.py +++ b/tensorrt_llm/_torch/pyexecutor/_util.py @@ -241,24 +241,15 @@ def get_kv_cache_manager_cls( "periodic_snapshot_interval.") if config_utils.is_glm5_next(config): - # GLM-5.3-Flash: one Glm5NextCacheManager (a - # MambaHybridCacheManagerV2 subclass) owns the KDA recurrent/conv - # states, the sparse-MLA latent pages, and the per-sparse-layer - # INDEX_KEY indexer buffers. The indexer state is a V2 extra - # buffer, so no V1/Mixed/Cpp manager can express it: conflicting - # knobs fail loudly instead of silently selecting a manager that - # would drop the indexer cache. + # glm5_next indexer state is a V2 extra buffer: reject any knob + # that would select a non-V2 manager. if use_py_mamba_cache_manager() or os.environ.get( 'TLLM_MAMBA_MANAGER_PREFERENCE'): raise ValueError( "glm5_next supports only its V2 cache manager; unset " "TRTLLM_USE_PY_MAMBA / TLLM_MAMBA_MANAGER_PREFERENCE.") if is_disagg: - # Only the Python NIXL transceiver moves the KDA recurrent - # state and the V2 extra (indexer) buffers; the C++ - # transceiver would silently drop both. Same rule as the - # hybrid V2 branch below, checked here so the message names - # the model. + # Only the Python NIXL transceiver moves KDA and indexer state. backend, runtime = _resolve_disagg_transceiver_route( cache_transceiver_config) if runtime != "PYTHON" or backend != "NIXL": From 70017b85dd4e83ebeee7e926aa2cf281ddae0a75 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Sat, 26 Sep 2026 03:07:22 -0700 Subject: [PATCH 32/35] [None][test] Skip MoE A2A CFT regression under CUDA forward compatibility Creating a CFT logical endpoint needs kernel-driver support. Under CUDA forward compatibility the newer user-mode driver still exports the cuLogicalEndpoint entry points, so the existing gate lets the test run, and cuLogicalEndpointCreate then fails with CUDA_ERROR_INVALID_VALUE. The worker aborts MPI_COMM_WORLD and takes down the whole test process. Skip the CFT cases when the loaded libcuda is newer than the kernel driver reported by NVML; real CFT failures on a matching driver still surface. Signed-off-by: Ruocheng Jia --- .../moe/multi_gpu/test_moe_a2a_workspace.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py b/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py index a857fb587c76..1bc8d6370d9f 100644 --- a/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py +++ b/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py @@ -5,10 +5,12 @@ import ctypes import faulthandler import pickle +import re import sys import traceback import cloudpickle +import pynvml import pytest import torch from mpi4py import MPI @@ -81,6 +83,35 @@ def _cft_skip_reason(): ) if result != 0 or status.value != 0 or not pointer.value: return f"CUDA driver lacks cuLogicalEndpoint{suffix}" + return _forward_compat_reason() + + +def _forward_compat_reason(): + """Skip when the loaded libcuda is newer than the kernel driver. + + Under CUDA forward compatibility the user-mode driver exports the + cuLogicalEndpoint entry points, but creating an endpoint needs + kernel-driver support and fails with CUDA_ERROR_INVALID_VALUE. + """ + user_mode = None + with open("/proc/self/maps") as maps: + for line in maps: + match = re.search(r"libcuda\.so\.(\d+\.\d+(?:\.\d+)?)", line) + if match: + user_mode = match.group(1) + break + pynvml.nvmlInit() + try: + kernel = pynvml.nvmlSystemGetDriverVersion() + finally: + pynvml.nvmlShutdown() + if isinstance(kernel, bytes): + kernel = kernel.decode() + if user_mode is not None and user_mode != kernel: + return ( + f"CFT logical endpoints need kernel-driver support; CUDA forward compatibility " + f"runs user-mode driver {user_mode} on kernel driver {kernel}" + ) return None From 4fa804d2105cb10b53c9d17f6001f2a897823878 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Sat, 26 Sep 2026 08:32:25 -0700 Subject: [PATCH 33/35] [None][test] Gate MoE A2A CFT skip on a newer user-mode driver Forward compatibility pairs a newer user-mode libcuda with an older kernel driver; compare versions numerically instead of skipping on any mismatch. Signed-off-by: Ruocheng Jia --- .../unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py b/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py index 1bc8d6370d9f..9b4bad7ec9cb 100644 --- a/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py +++ b/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py @@ -107,7 +107,11 @@ def _forward_compat_reason(): pynvml.nvmlShutdown() if isinstance(kernel, bytes): kernel = kernel.decode() - if user_mode is not None and user_mode != kernel: + + def version(text): + return tuple(int(part) for part in text.split(".")) + + if user_mode is not None and version(user_mode) > version(kernel): return ( f"CFT logical endpoints need kernel-driver support; CUDA forward compatibility " f"runs user-mode driver {user_mode} on kernel driver {kernel}" From 2d17aae8a2d94d536b1efd1bbdec0293ed5bc044 Mon Sep 17 00:00:00 2001 From: Chulian Zhang <851104+zhangcl@users.noreply.github.com> Date: Sat, 26 Sep 2026 15:43:32 -0700 Subject: [PATCH 34/35] [None][fix] Gate MoE A2A CFT on the NVIDIA kernel driver version CFT logical endpoints need kernel-driver support (615+). Under CUDA forward compatibility a newer user-mode libcuda still exports the cuLogicalEndpoint entry points, so the existing gates pass, the workspace is laid out for CFT, and cuLogicalEndpointCreate then fails with CUDA_ERROR_INVALID_VALUE. Check the kernel driver version via NVML in resolve_can_use_cft(), which both workspace sizing and construction use, and fall back to fence when it is below 615 or cannot be queried. TRTLLM_MOE_A2A_FORCE_CFT=1 does not bypass this check. The helpers match the ones in the NVLink one-sided overhaul (#19610) so that change can take its own copy when it lands. The workspace regression test now skips its CFT cases through the same helpers instead of parsing /proc/self/maps. Signed-off-by: Chulian Zhang <851104+zhangcl@users.noreply.github.com> Signed-off-by: Ruocheng Jia --- .../communication/nvlink_one_sided.py | 54 +++++++++++++++++-- .../moe/multi_gpu/test_moe_a2a_workspace.py | 47 ++++------------ 2 files changed, 61 insertions(+), 40 deletions(-) diff --git a/tensorrt_llm/_torch/moe/fused_moe/communication/nvlink_one_sided.py b/tensorrt_llm/_torch/moe/fused_moe/communication/nvlink_one_sided.py index 90ddc3405a65..101b4316a644 100644 --- a/tensorrt_llm/_torch/moe/fused_moe/communication/nvlink_one_sided.py +++ b/tensorrt_llm/_torch/moe/fused_moe/communication/nvlink_one_sided.py @@ -25,9 +25,11 @@ """ import os +import re import sys from typing import Callable, Dict, List, Optional, Tuple +import pynvml import torch from tensorrt_llm._mnnvl_utils import CftMnnvlMemory, MnnvlCheckpointCommunicator, MnnvlMemory @@ -57,6 +59,7 @@ _CFT_MAX_BATCH_FOR_COMBINE_ENV = "TRTLLM_MOE_A2A_CFT_MAX_BATCH_FOR_COMBINE" FORCE_CFT_ENV = "TRTLLM_MOE_A2A_FORCE_CFT" _CFT_ALIGNMENT_BYTES = 16 +_CFT_MIN_DRIVER_BRANCH = 615 def get_force_cft() -> bool | None: @@ -68,17 +71,60 @@ def get_force_cft() -> bool | None: return None +def _get_nvidia_driver_version() -> str | None: + try: + try: + pynvml.nvmlDeviceGetCount() + except pynvml.NVMLError_Uninitialized: + pynvml.nvmlInit() + value = pynvml.nvmlSystemGetDriverVersion() + except pynvml.NVMLError as error: + tllm_logger.warning_once( + "CFT counted writes disabled: failed to query the NVIDIA driver " + f"version via NVML ({error}). Falling back to fence-based dispatch.", + key="moe_a2a_cft_driver_query_failed", + ) + return None + if isinstance(value, bytes): + return value.decode(errors="replace") + return str(value) + + +def cft_driver_is_supported(driver_version: str | bytes | None) -> bool: + if isinstance(driver_version, bytes): + driver_version = driver_version.decode(errors="replace") + if not driver_version: + return False + match = re.match(r"^(\d+)(?:\.|$)", driver_version.strip()) + return bool(match and int(match.group(1)) >= _CFT_MIN_DRIVER_BRANCH) + + def resolve_can_use_cft(can_use_cft_counted_writes: bool) -> bool: - """Apply the TRTLLM_MOE_A2A_FORCE_CFT override to a caller's request. + """Apply the TRTLLM_MOE_A2A_FORCE_CFT override, then the driver requirement. Workspace sizing and workspace layout both depend on this, so they must resolve it identically: a caller that sizes without the override and then constructs with it would lay out the CFT region in an undersized buffer. + + The driver check applies even when CFT is forced. Logical endpoints need + kernel-driver support; under CUDA forward compatibility a newer user-mode + driver exports the API but endpoint creation fails on the older kernel driver. """ force_cft = get_force_cft() - if force_cft is None: - return can_use_cft_counted_writes - return force_cft + requested = can_use_cft_counted_writes if force_cft is None else force_cft + if not requested: + return False + driver_version = _get_nvidia_driver_version() + if not cft_driver_is_supported(driver_version): + if driver_version is not None: + tllm_logger.warning_once( + "CFT counted writes disabled: NVIDIA driver " + f"{driver_version} is below required {_CFT_MIN_DRIVER_BRANCH}.00. " + "Falling back to fence-based dispatch.", + key=f"moe_a2a_cft_driver_unsupported_{driver_version}", + ) + return False + return True def should_use_cft( diff --git a/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py b/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py index 9b4bad7ec9cb..5e6f9c8ec1fb 100644 --- a/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py +++ b/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py @@ -5,12 +5,10 @@ import ctypes import faulthandler import pickle -import re import sys import traceback import cloudpickle -import pynvml import pytest import torch from mpi4py import MPI @@ -18,7 +16,12 @@ import tensorrt_llm as tllm from tensorrt_llm._mnnvl_utils import MnnvlMemory -from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import NVLinkOneSided +from tensorrt_llm._torch.moe.fused_moe.communication.nvlink_one_sided import ( + _CFT_MIN_DRIVER_BRANCH, + NVLinkOneSided, + _get_nvidia_driver_version, + cft_driver_is_supported, +) from tensorrt_llm.mapping import Mapping # Match the neighboring MPI tests: workers must receive this module by value. @@ -83,39 +86,11 @@ def _cft_skip_reason(): ) if result != 0 or status.value != 0 or not pointer.value: return f"CUDA driver lacks cuLogicalEndpoint{suffix}" - return _forward_compat_reason() - - -def _forward_compat_reason(): - """Skip when the loaded libcuda is newer than the kernel driver. - - Under CUDA forward compatibility the user-mode driver exports the - cuLogicalEndpoint entry points, but creating an endpoint needs - kernel-driver support and fails with CUDA_ERROR_INVALID_VALUE. - """ - user_mode = None - with open("/proc/self/maps") as maps: - for line in maps: - match = re.search(r"libcuda\.so\.(\d+\.\d+(?:\.\d+)?)", line) - if match: - user_mode = match.group(1) - break - pynvml.nvmlInit() - try: - kernel = pynvml.nvmlSystemGetDriverVersion() - finally: - pynvml.nvmlShutdown() - if isinstance(kernel, bytes): - kernel = kernel.decode() - - def version(text): - return tuple(int(part) for part in text.split(".")) - - if user_mode is not None and version(user_mode) > version(kernel): - return ( - f"CFT logical endpoints need kernel-driver support; CUDA forward compatibility " - f"runs user-mode driver {user_mode} on kernel driver {kernel}" - ) + # Logical endpoints also need kernel-driver support; under CUDA forward + # compatibility the newer user-mode driver exports them but creation fails. + driver_version = _get_nvidia_driver_version() + if not cft_driver_is_supported(driver_version): + return f"CFT requires NVIDIA driver {_CFT_MIN_DRIVER_BRANCH}+ (found {driver_version})" return None From 1ec4cfe91d7d36bba87e564b90c79c134dc2cbe0 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Sun, 27 Sep 2026 02:20:00 -0700 Subject: [PATCH 35/35] [None][doc] Update GLM-5.3-Flash 1K/1K performance chart Replace the four-configuration comparison with the Pareto frontier of the measured TP4/EP4 and attention DP4/EP4 configurations with MTP, drawn in the style of the GLM-5 guide. Note that attention DP batch size limits apply per rank, and extend the benchmark concurrency list to match. Signed-off-by: Ruocheng Jia --- ...yment-guide-for-glm-5.3-flash-on-trtllm.md | 18 +++++++----------- docs/source/media/glm_5_3_flash_fp8_perf.png | Bin 169957 -> 73606 bytes 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md index 98d1f5cc4403..dadb30919a55 100644 --- a/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-glm-5.3-flash-on-trtllm.md @@ -92,7 +92,7 @@ If you want to use the latest main branch, you can build from source: [https://n ### Recommended Performance Settings -Use these configurations as starting points for 4x B200 with FP8 weights and a BF16 KV cache. Tune the batch size and token budget for your workload; the [Performance](#performance) section lists the settings used for the measured curve. +Use these configurations as starting points for 4x B200 with FP8 weights and a BF16 KV cache. Tune the batch size and token budget for your workload. #### B200 FP8 Config @@ -133,6 +133,8 @@ EOF Attention data parallelism distributes requests across ranks while keeping the routed experts expert-parallel. Set `enable_attention_dp: true` and reduce `--max_num_tokens` to `4096` in the launch command below. To enable MTP as well, add the `speculative_config` section from the MTP example above. +With attention data parallelism, the batch size limits apply per rank and each slot reserves KDA state, so size them to the per-rank concurrency (for example, 64 for 256 concurrent requests on four ranks). + ```bash cat > /tmp/config.yml < bench.sh -concurrency_list="1 4 8 16 32 64 128" +concurrency_list="1 2 4 8 16 32 64 128 256" multi_round=5 isl=1024 osl=1024 @@ -443,12 +445,6 @@ $$ ## Performance -The chart compares TP4 / EP4 and attention DP4 / EP4, each with and without MTP3, on 4x B200 with FP8 weights and a BF16 KV cache. Measurements use the `benchmark_serving` client above, ISL 1024 / OSL 1024, random token IDs, seed 0, and greedy decoding. Each run sends `5 * concurrency` requests. Points combine all matching runs; faint dots show individual runs and bars show their minimum and maximum, not confidence intervals. - -To reproduce the curve, use the serving configurations above with `--max_batch_size 128`, `--max_seq_len 8192`, and `cuda_graph_config.max_batch_size: 128`. Keep CUDA graph padding, chunked prefill, and the overlap scheduler enabled; use a cache memory fraction of 0.5 and disable block reuse. Set `--max_num_tokens 16384` for TP4 / EP4 or `4096` for attention DP4 / EP4, with or without MTP3. - -The horizontal axis is `1000 / mean_tpot_ms`, excluding TTFT. The vertical axis is aggregate output-token throughput divided by four GPUs, excluding input-token throughput. For repeated runs, throughput is total output tokens divided by total measured duration, and mean TPOT is weighted by request count. - -![GLM-5.3-Flash FP8 performance on 4x B200](../media/glm_5_3_flash_fp8_perf.png) +The chart below shows the Pareto frontier of output-token throughput per GPU versus per-user output speed on 4x B200 with GLM-5.3-Flash FP8, a BF16 KV cache, and MTP enabled, at ISL 1024 / OSL 1024. The frontier is the efficient boundary of the measured TP4 / EP4 and attention DP4 / EP4 configurations with MTP: no other measured point is better on both axes. Other settings or workloads can give different results. Benchmarks were run using the `benchmark_serving` client above against TensorRT LLM 1.3.0rc29. -With TP4 / EP4, MTP3 improves single-user decode speed from approximately 153 to 372 tok/s/user. At concurrency 128, the four configurations deliver approximately 5.9K–6.3K output tok/s in aggregate. MTP acceptance and speedup depend on the workload; these measurements use random-token prompts. +![GLM-5.3-Flash FP8 Performance on 4x B200 with MTP](../media/glm_5_3_flash_fp8_perf.png) diff --git a/docs/source/media/glm_5_3_flash_fp8_perf.png b/docs/source/media/glm_5_3_flash_fp8_perf.png index de0e510579f816e4d34aaea9d24f86255b768c00..a660575017db4a37d5a968bcf3e19f737d05517f 100644 GIT binary patch literal 73606 zcmdq}cT`nZ+dc|oj2aa%Rs<0e1#A=%DN>CM=~k*hkR~0e(v1b8C@2U>2SM7V_adMo z5~P<+*)$Q64$?cEYpTine&ajm|KB-#jAuLqH+!!&*PQo#wK*SLP?Toa#IlKwj*da* z{FzI1bnBkc(XAEzWj%hg%kRA({!i5Atg6jr3j-T_9V>l01sxkpQwtkYqwBxf>04PF zS(pp(^7HfZbN_b3#>UcGl#kEszjyFjSQ+wJ(bmr4Q#M+jSF@(0+xa*7-`)$LTd z)?KSQFfg#NFqASfsw5}Z@a)iK+Fo6gqJ_aDp&~b5oGyD`S~~E_lW(;CmX^!(Rcdqd zp3%vf8UNzq;yZWlG#0zLadL7V;^K;ttLeI=QLHT;V&CZyguhg9Qdagan)|dlc&4$j z(R|UM-FHQ)dZ{DJ&{SJnd)>Nqhs4FNm6w;x%E}JbS}Q0jdZnkQ+YB}yqBb_h6%}c% zS+i!-#*G)9onK)P5}!PIV(0Cd9V+f{Dcofyu)FB*zklITOMmxrT*PsCes1oct}gAr z{`%|Tg9jI{Uw?63SGUcj=#UDebK?VT6lc+{be)9lbsny6>Daz9eJ8sTH;n$ zW{XEQH~pb>{RT=PsU0mX?+j zmpv>jQ>9GW{rIxx_{*{CBSRx2Nl*7n4}Lh3V$(mr-pk0;w6Q@+DygzkzGQjh#*Ms6 z2kpNs3^k{O<4cm}C;KXDYHG}j$BGseDJm(-c6Ql^j~r=ZZPq(x^v{#u)iZBAt(IDK z_UJLXaieySN|~;cW{*!yOr)lz6pV_#jE=tbu0^6W*rLd2a*=_Nv5BgpJreV`=zGnS zQkFe?3O(!J$lYwLROnZBqZ4FRngK)PSVIR9%@R; z?|y0_Xr!$PxLYtrqQ+I>gCy8 z=OTs7Xwe0u`Ev2`@pg+6OJ7Uuwg>VE3JDoECu`CQopM~~O4yCx+*zl|Y?m@!EjA{y zB-YQU7%rLAtnH?$?Y5jXn%Bp6_RN{xGh(hwxr}>7KYz_N$WZfO+*==c+T*y(f=!22 z1<%CHjQRBQPoJ(YPp7zD#ps^amHPU6t$MoNgXH|d1fiarUs@!}-5K^=xpGC1x{H~) zb!6KIul}TzQr6|*9^T&GUdNeM!z}AQMGx;oyu3{bS1Fl})5e-NOLwZjx^*-5`mLd) zoa3%bb3)-6bgwURf7n3Bd*$-wC}}o_GknEMRt3K0JSUDtwfYHJbSC=v_$V(rFAT&c zZ;bKyP+8gNt0XxvdX&y%=gyr27H*5dR@T-GvpGUXvITJ`DMbqj-Obi@lG0wRbsp@l z{i8=i_KAsU)QC?$y8ON~Rp)fDKHY~8AEs(gZ^;V$DiP3X+T=7=7%SZ6++d~~wKzDE zQc7D0?qO6@Q(IUbbL&dho5&j+8k$Ch8)9Fb^XRZL^7^Nuq9NaQNYq)JF7q9|KAq*$ z_wUawPlm5HhDkUL&pi11?^=hx(AekCccYlbWoIknQa_ZJ*Nyj-jktzVw;y*i>*%bE zO;11OGFc_e>-E7pP@apEi|g>ALo$((k%JAf-tt_&4?R7zOqzKMw3ktbi;o>Uc5NoB zmwn}{2cw^ijI6BHAIEQOFL_w9($Tq$B#~*(wbR{GlUhb1xts8S&b)w8n zP1zLGc+}O^<$ryqflT4N*Az2V%p8mRcRt zty79|OTB%4COz*TyKDd6m17ZJQWJ=#AgiEowb*s}^Ux4Qz_9uT=ACN^K9+b;_Tt7E==L zycMP3WS>;gz)77@6UUi)7_OSiOs;Ub5)wXThxXlXe$Anm&Uf*8h`1Q?(q|UF3 zLC0mhB^Tlpy1s0ou`0;Erjbwedq%%yi-oma)PbYQF(A~F4&P7lK3yd1L zB$lTdgmhC>(@wd~S7=`owX=Wn?Abey=Sls8&DXWGw33oJFRG{*3P*}NOz2^bW)C@^ z%l~BIB_=-q+1JjuoU}22Znf)|@4U#(RjrX+Oq?0*FtM&gbvUl~&#zTMf|S$aJD8YE zrU$8^yy0u;wgHS;KJY!DG+4n`e7v2Bojs)sJ<%-rwfV8YXFfhJo;}+^Dg^;1+xMBV z^YQUf0?qXG^$ExjwEiMbab;j&5L#XJFUzNqZf4qYG$*7WsWjx(t5=zSzWK>zaU@67 zaVoCLRVggK^XjI}n{7Y6+qiSru7v35GZ+|SV`CozKknY$xn~b$&D!;uV%^#oE<`+f zL?0^YqVDJCmynX8a^b?07cXAmdg}EpB!>nEGo5D5d%t{1c=6)Yn>TM*L@XoRx3G^s zyT4Q7uki42^P+i!<%Q;AA@1@F`!Ov_xW&zzY26~5cX6i%N(kBws}huR-(4TH=1 zL+Y)0*0BUuRb-dz)hoG9Y?qk$XFsF&MRoOr*-odLFSC1p{WSuUg_f#pXniMJ|5I~Y zn}&*}t=~^;*S5_|ojCIQ?|D%UR`cb{KBJbt;N|0N*Hz zwov8jq@ARlVenXI@v4fZzICBusOVL!f|vCT4M{Ym5D^`b0$QN1*dP{S_~77RB8@;S zTxi>rgiZgGqU9M4N;cZLaeMAfA(tGBF7vtW#jIv66PbgeR@yEoK<0z{;=X-8y>m9GXVqYxl#qlm6on!LyAtqy; zg=87h5}Wtz-JA4a`vK$8_PmE)UUnig2B^Oa!x=SG=+e8JWV|0eYOE6OYX9P~spG1t zY2v>9`x&<1;wcU?O<|SR(~H(|I&1^!^uQzQVvNQft;MtELth=Cu*D%vi#9=_#O!#qWX-gS@k_H+5lM4fwlO#WbVV`EFE;PK=A1!F}SMza$W z^(|?-3^n`!lfls`HZx;=c{1AQ0BmJmdou!=^W9cmOYTje444+F`2_?R&b?Zm?+b6M ztbF9<)ubTYq1S3YQe>=n(4s9{VZ#=-%sp;qW=Y+HIz8BRP{6( z*Cu;FF{D3za&d01A7wzeJqM4xWM*Y~F~{Mt;DL&{%1ROe#(~`q>CAtHLt8Fr~$g379rW!YRAcZPi7t4G+c!m6_67%;U@O`|rOK ze5cX%F7aAk{bC=sCQLkRLn=}9HAAJQ~goi1nRL!IA)gU+8^ld?sm5rrBEnI*}lHM4Bj=s8vxyAv|z(MVMl)Z?R8|m zpcx=w9r+eZv${KXeoDHQcWIB1X>F)|XY1&exptSNS96~{l^?vs5Dn2xQbuZ3n>Fo2 z8QCQhb<4cevWihNSOg3nt!H4-Giu8+3D7y_@cEyoniD%sqfE|R@{nXTE{}_ex0xxq z?+;={U$N=k`}ZHZy1Fb`on}v;JLg{U-rT#p#G{SlnNpZ|e0}{TP!+YTJ)Cyl7!P;% z^^`65uySla|Fbo?XkM#*q zRn*vcCC6{@s~#zDIxiAly}EWpInnZ9pmTCeOpIO4bw`liy^c||#>VKraJSCXCKIJU zE!JfCYo~-B)w$EHVlK7x^>py(*9XF#ujkUWCaPu7R~*ii*42$#8TWRxxlvae62iJX zpyXB`C}2pl4G~N6vmI(O0tfInLuc!-YcsR+eQGv7K3-SjwqhOJTKf3$V@H%o9{sHZ zrnXzxw6e0Zu_d=z6nBg+tns21*rx!180K0m&5eh$6aXoKnH_I8JieHzZ&esr*gEU8 z?6?g0s?51D;X`FZ+{o7 zim3BkYKJ;?WZv?PmbTN&+(Wt<=^8oGfl4?Ch2`&h8dtBI4+HsWjW7JC>{_S1~j(k*1nvC zCaa?p30zCRy>;yd#zsE3c{vZ`96LYMpscEUp(!aTq$}CBo^Ww-0n|!}i#vGV-TknL zh$_f^0)=uJf6mAz9)}SQ?&)-y)vOAWXdAT8LZ=QE4X?BY@gG;KT8 ze-Wa@+f?z#`-9xPQObkx4ACA5Qz=oCMfsp*nWzN66csJmTU-F(4DEwu)Z~P(2);9? z!!CGbzQ#>`xpMQSO;bbKtLh~lTU5@U|C_82IR<)`Haj3>(R6<#qiG74qMaa&!< zpZ^j({n=MZJu4iFMM~jJkol|ebA2(r zF_k363iY=e0vvTu zdK89Vl?$73?()_4%!+}Eajm>l^gw54Cx+Vk7OT@{7XN?%N`5}A``90oeKpO7;VuSQ zc5gtOGA+AL<*bCz*;o)TUjCt?f}w5g+O=lNiRQ`|F8rA@L(6;rcuy=Kgx%qyWXp2S z0M)_={J~ZJDGgcIej9kZneR$hkxRz}7}H_dwd?6GfNZ>d`*z2U9nljLR(nM(Uw<{p zu5Z2f8Q*mrRUr0VCXjAPt41Ho(LZmaf?QV!7D{T!;BOmhKU*+7K07<+6Rn4joOWBC zbpyjOvrY>P&~|N&S<#n~dEmt^Wj?*rDe8WgkFW1Eo*c7lZzc$Nm+NAZVwU@Gi?bgXv zOs7PHBTY?B##kfnx+sPOAmlF$QpCh5*=qXwFNyF_;vyHoOVtn1G@#mzmog2ZP-G~~ z-vwSZ>niFnXVBQ6T%;Tn6f{*SP^0RiD){H?+jf4oMpd87eY1d}g4Oe|4)TmmOc5TB@%C9BooVsL7|N zLQzI;IXDcwnyk(UeA#yFP`mP}`F5wuIEzb{E_v+g5Vu!eK$kOhHOIotN8f3aFt4z_ z^QFB=k`Fj#xM9U7Zu@X< z^X~O1kuIm3A#iCrkCz4sTZDCU_c#5#ZrwnWhD8>$)68(I`$qQ}f0?fwebv7T23H-1 z9MW0n#9(+=t{%c2y-AhqB{aL_!Ws3#8JdY1Z%{w9FeqFmP;o*Px#SF`jviYOdL8k{ zT~nX(`c@x-04>M(PKUm|!7d=xH$@_cy-a$eyd~3vBnF@vhObP;YUamvr|VZdd-_zz zb_|#$m?MJ&xWK{PbH>)MzrSCx>Xo^%as9)cN9b2~EiNt&H5adBb%zf<^+SofhYu1Hi|Gml}Z(oxd$@%GA1V5X&6P`W83gIDC#fj zb43K}&bC28cp%U)EUnf4OC>Y3+DxbJq~vW%_?lbSGz{4YT!nK$b81xto5Gdnya zoD)hm7436`_YO7a;loXo5y)P&sa6OqE5HOpcQ*50fMg7cPeC73<6I9;`Rv)V#$;>n zJ;K#rU2_7?k5t#J@v8jQY}eksX7;;mTwRO8TxQft&iD`!C`mm_mPi3utTF6MA2);^ zUZZO<^BGTim`%W_o}=WRm5BhqhShYI6*y_(@=TkunTnQ!enHB3_T2pZ0LVb&>T=g= zR#4n9RG`t^9>!R|Z9dW9IRoVf!W+?_Y{6`D7JfIE6BHD5{Pf@e{Sem%%s1iUc?F7_ z%5W$p%Stqf+9Dww{~6D);zmp?BrYH|!a~u{)P2h8Ti5AFy?DWGhEEW+8-9*O{46lg z=vJuH^<}+s>+rGAdG*TB{6PyZ%5InEJgwU84Pi|SZDJl0DDI>WsB7vLnkb; zbLz3N;pdjlo3J;m!Gf{@5IGEu-as3&8FV!%RpX~&CJ>NGRe?Nk6{3Je@p98(QDRC; zGXPi;B+(A%$*R3xHCht1CgahWu`X?VoltQxt{%3O$F_Eq;!dYA2}<=3ZjyV{1>(fRa?V|R`G*3^@>3{Zn z@8)%FPSKXa?_N~}Rn8Q-chamiOg7G0Zx5(uEhvP9w1bV!+?t)ZKEU-^@dw0h`Y)W7 zmez2YbNKr44`|)HU4QEt88MrSj5zQ$vv1KrqB~2~Fmpz@%x~B)Aub`|67WEYfZqt! z!+eA_DG+c@IZ0hVi}ww*Ot7NA?7Y{}eQVRhJaNCv5VV~*z6*jIbnUuy7a}UPrk5TX zx7`Uit7%hrb^&URVXoA%WA(<7Y)YSddtaikmbChulSxTFYCCwPWCjA{au56J(ZrM` z3{@U?q&R4jUDfbV90o)JvQnrlSDSSOPtL(($1a7tF2r|?2L}Z`3kZmP^-A2VJ@+E< zo(R>dc%kT(8u0w4ZO4#7@rx4!-f7^u+@tCSDnr zl_nT4!3D}x&@p{k4`0O@1BUDwojd4Vh~3!_zl2ecpvR?NV_Xdirg2~9r0+uOUlvLzxSLeZ^|)|piZ07|rZA(`G#G5f>& z_eWLTx3sphngTAQL`1wc&^9(s7%f^f_ZXa~1;3=xy0$t!h$+ny&ZzaR@7(Y#dDJuiIT{%f6g24HcU1ZoZPFzp?5lcWAvE{>p`ioq6JzHM z=jP^iHyP#3F5M6chLF|!^+o%+&GoW|a-FuH+tz{0lQHO8nGvSUz!B)sPdTz?iuK#m zUC+oGC+V`VyQsA-_hyvNsk}U|p*(SkNbb<2nwf7Jg)MBwu(qxnC*EqEDy60!;N%=Q zAT|F04M(bVSPYF4!kYNZV>@Y6T4s4_XKuM(dZjX?x~wHOMQBE<0o3!8&EiaaT8Wwk zb_2DMr{Q)r!Tz~&>C*K%F}@>58qgY&rIxz4Fcyzne`%hTP^e&jAt@=jY15`lKzQrv zch`Oq=7Zow&(z`h-~s=&gGUY?JP(|fN4QqYa9g^*9Qfjl*XE;+;PW#`!C=N1-3 zm)3W6G(Ws}$fF5;T(#=(USac)j(akvPv0f$tw4Vl%%kb42Bo1er_q5cdE55G&}+g| z$b^y?>;i?>d|Im$wS}O6mX*;1^3Nkkn`Nvy%nBJ$$H^q)+z9nPJAS6tln3hC-@ABf z9#T?5a&orpHuP5$tRh${%8+${B{)Jg6O$t&-z3PFJ zDjHz=>%9HBWFZ^KpzhAP&566V4pjr)A2*{N4_>8&hik{~%Gx7p6Gwey<}_T@ZgmC3 z2nHd;EXcd9@vv9wY?d($P0|?erBhtbRa=3-m(1)j&wX?BO|IRDI)=CEr=Na0e)H2A zg%FXtdSYlo#2o1?)UbFp?oO4j1{g@vDpa>`zbRc%-LS{TZpIokP3JAYs3>iee+J5< z|B1Jq%TN6L^zHvD1hxofPW4gOoXQY(7|Ck3nd+}oUL5WyP$6zt_gjxpYL10=$qc;M zg0e)797VpOxfeQnKb`g5H4tQyY^3r@P*PG0s6?oI0Cq$iG?}lI!I@prrfNe?-yX$v z3Sr20!>Q^a{5(8WKL4=d{y8JBp-D{-HSclNczv1a^P~;T*)NKt3-1-A9h{s@a24*p zf4~WdE%ju|nO|+i)9)z#Yp*6x8kcB8lIniIw?n1&pO_M;bTeK#ho!; zI)wVeHXA@`yDBgLDEP&YI$=MUrbd32fu!OX7YfRqKi_Y;{7J%T<_5HXM$UU9erJvb z%G-6_a)V7E^k5T;X+e1hF*tk5yzLL3sH2_I=Wz=Qoo9CVDJq}p1NZD!iJi3JFw@SR z@u(@IKdBSucoQX8h64NlJpe$}rpb@bbFl>|synh zIi1F{E#P>CQ}uH0i;&92LN6%F$kgTD?8!81%T|26Ho~5y8+d<(1xr+&B)AspRwDb*3UpME(z79yUD8$Cr~yc(S>tApncxRGqXSnc!jKt(V#+uc3r3 zEKSxhxJdyxy&dNHZ5nnhjbpEd+(9X*uwc6lkbREB9w~S*I?|RigwF8I*XW9*IfPYe z4g5vqW#jSGpdgmQ@%MXZ8HV1cd;Fh0>)+xwcNZ=~GNf~p;g&Ra?Z-Ta4qb$}rchA# z=&X$viXIr5nE@yI?En$m2~D6ZPBq3~wi~N~vS%;i1(M9ddl*bR7Bft(!k;JOVGo?` zIx^oxnt-@OSv@kULd8@7osBcJFJJc6a_HR|JoELdC8hPbv`8&aT`pNv_38G=W_SMjmkvnT;_n?0 zP!o-%_iUu6Z-nTo+aPH_dM)@~n2pd=Nm8h`(19wO1>Z}wv2#zZ6TS*rO8U`m{<%L-i|h_~NQ zMe8EUKe7Ba`oZ_rH!w)Z&21SShiZ(siT|Ol9+Yj~Vcbw;)hkQf*#{3EK<~P6?%X*F zC9Ew2V){r|u{Ihw{3Fb`3=V=2ZGj3(m`##+#NE9nA;=T|LZ@VXunPfAgwjIBIRfd5 zsL!g`9 zz^eu14|?3Yx7$C=s0s86v!eb>@Ol&+B1FXyi5Ya43KnXdRU8p+fK=K|A>v{KJJEP0#-?_Offoh&Vj|1s#tGt~O7gt+2+ex_? zbuL~hT#^uQop??}et0ypE5C*S_3l6SXt$1tZhj!HmulKWj2ThUd}@A;X5f_PI>G8rc=wuB9_0aWY;T0!pF3kM2o<-(2EXUGM>7W$y*t zbc7=3GYK`!4VYvaJO^$j!xb))8QYs*I>dA3|YnX9W$L2;m_r9&jW17r+yBmvzNz zuo#P07KW4oKfRmHaypvgcjp(7AcN(_tRu;}ry!jB+XxMGif7DRYs;}{MnM;JTXFWN z>Fe)GH$YRems5*Jo(SQkhA)No86a|5rQ*N)>}gdp8NP* zpR9_2>zzWqXQk4l#0K|3XW*|7A7q2Ae#7(`;@_yBtBoq|&!?RXL!>hG+)Y3%P^7V1 z)y+)2jaUP%FUK_v_Gk?SCLyTjCrd2to z5PPDC;}ULr^D_9SY4fKDuYNo#@1>?abj^RdAEV2CbDv2o?q<(>Bb0U;kU6FogP}nf zLB6Nm>FZK;N^IV&W14EochfX`dvD{hKzX7yW2|V24~Xe#)DU9?O>xtfEn^e6b^(0j zt_R`V*)hEkQ$a~0K`D5rv1O~(HTY2iIiCdtM@&>VWaaIeZ_l&dtM!ZzQ9I1n6jU=J zq?&c)XHf&IR#sM6PFO|n;=LA!^q|{|*Gu*|Os9c|N0MvrZQMSExXB>w#LEymb+iE_h)PBDLtEUE&HSFF(P&uqN>a<2 zRb$*ok6y^>axGA}(^*raB(>ODW3#$EO97Cs??m~BeNB97Nkp{KFgXp=Z0XlB^R$aYB~! zp3AYH#;2#V77~gJ`61YdUaLq-O|7-G9k_Sz9jtg0Z)JGM_>ZmI`>FezsAbbW|fu&{Y!t_uq%m#HI`&h-$`ee-lQ!hCM1uFCu z^~WTphNXA`wmHZkGK{(>q_15IadvSjm|8_B(NI+}5)hC@*!&u$&)5^$m}=IxFyxgg ztaWT{vtSq^auoaVINxz@3-f)fO<^22$2*ck)617O!Ffb_zev)Z^xF7hE zUhWufn59WqK}ktCL`@v>x3XcxXlq8<*Sn!;d&WT6KD61KO5`ChXGtn_b!Blkr*i1P0cp$uJiaG6IXS0Jo!YT$R}9Ic%&SMgtiHzXG&@Qy_dUQc%fK$# zZXBsZcjMguB$l6qsodN8Bb@wk2S~_|pa0LvJ+s?yAcy5lTDc=C-Qr z&Bx8%_@yeC7;<>vl!OFXfNZh|{`@n7ECO)8()aHVL%%2HFD$l(Pvabj0Fitw+Iw%; zix>MKw*?n@OReN$cD1=Mg0>>wY6KsJNSc5g;Aa%1&oI4>VQjOA*(E~N9kQ&f8aFUB zBzZZK*NuAetWpuEE92{qKbaY@JQ6X-Cv+Uzb@v0K+c1`nMEWJ5eq)kV6 zyNATjZ?nZ>GUMyr7awgqQH8WG!H73dGm>TmtXlO0j0WmoYO;98LJCuw8lip3><)62_HgcZtR$9n2|SPU>Jz+54c?REQ*%MJ+$xb_HywXcxPVNUpRR zhGzw{TE{>>8$^#VpUs=ZBmxN|?IOW~DjvNgbopPK9O&T$&1QdCS}`)RT%X*4COk^E`& zoS zJR>p%t-m~?l-N=r9FpJ{KY!hXg`uzK^z^jj%6uQmC1qGld@c`yt%?YE6hI)!Zy?mJ z?1KKDg{&-(YH9>^l7oZe28iLmnM>40h#Vw<(*W41e`^JQNWeR0f^Gk*55Av>dXYt% zM!2a3*o3;OLT^y@tiM#&p@8oM9QYBFL=uJs2^3a|;=0fuMbd{mckIy12**@y=S#9~ zCJ$HCgFM{+X+*{h8lW!Obh*run@Nxm6;&seJ&33+@=YZDPAo>OqiZ{p+qhLzPkJ)( zWgCt4UpKD;L^VNhj+W`YS4Llsy66_G!y5&6z>@DQa15hH`=&Lgs9bP+pd*zSx7 zPd@O9*~%Dl6(pb_3D#PO?2S+vu4E9JKqLGy(-}-2cTDEdd@RkAeHSNi%TtaAl)^HN zhCqcCtSbK@ERi-iW?wI(9a@!nC)nq0dOOYP+5-Yh7B8r{58yP_P12eVR>UI>gA51I66>P zXAwL<1JhEVJqL1I$*S0_r*5vcdS5b`gEnm6=b+&g>uiKAk>;3~owdL5sJdEo_;1+j z%FrrD)5lQ|r!l|nTW@*A0U9_u<|{Y>UBKp{_hy!K&Mlh%vL|>3Ys(mOP}yu`zVKaX zDRXwPe2kDcdT2AO+(eo`kA}ga<&W|vO^N)b?0|3QyjWYmjFUiNQ;K#9^l{1zCX-o9 z>K{^bZ?+`T^0cczxekQQZpMreC);Tvnm5zaXBt6&t3xJOsl&2_;o;-Qwx_R{Q4$kL z)*;|=^Tg0t@)LMZEo@9ohQfLJ7F>yiX{^-rCR2?*6J%@7}e`par?e6nA%bMn9H= zhYs;`ule&>psB&lY*MUMdD3^GSwm>a>)^M!S7%B!z^5^K`}+GEJij1`9RmO+iOKe( zOtamtXw6}_zpo?`&_y+D8C`+$iM+Cg?B<`>uRqMgqX3z&8ITx#XgJ}56KmAOMjzxE z^k#QENm-z`fo2tKJnjVXo0Kj}c6O6N*CS>Dc$0dwEfAedu%eaCq~^<&cCfI}l6Q+D z?D{UX0|Et--;Cb6Db9bO(V^8e!mF0sGwZo%iQ%DBI{Dx51c})v7cEW5c`T@4_Xyct zg8c+2gR4e$FU}H1RAH#oiMSHNv194utnn^1Di9$(MoX}jRzwUx&VPuIjx5_C`k&Au z8CRklkEWq>le(8gGpG)Uhao5GyiyI(P1w|e=gp0GagR##`b2~yE0?Wy&3a~yk_-Fs z+)mcRJUi%H9!+Fl%`(8-AuQf2P{>@TDcX6*Wtz4dIEHCRQ!gfQPS6B8A*4;iWEn#8 z0pAZt16D^4X^Kwh9a1R}UMlP$){t1y3FnpMKgU+vI)`f8+S*1XGg?g>-Y}hQ=uY&PX)0}Fy|CNnnMy2nn-iG3kcRslT9<&Ol5a5TH}aN2LABDgCx=vkY1o`V6XM)0eygTHecp)mO2wFct-i8lv`3 zGs)(I0vVGBi>1_D6vEpFh?zk5X9U!jpyNh9Tna%P1H<&y=!#HS74`_5AD2%KK`iO& z)vLV0wzJR=iO7Yb!OMyjGUHhCX2{We;ym_8l3-Df+491l`Gch<&CaXp>NNcx zzNT z9~taI|DG-w>c4&y5i)35mtY@P6%nU6@}3pi5)txpEs(X{`_OZ-^hB<9_27rnod6Gu z7f+)YMTFMZ*Y7}jAeZmx(c02F8;hq;pSox85(wezt9z>woQ1w^gj34I*m$Z%*PEHE z4BTLD+@m+|rRhNA0#{y9_KV%K}DM57k0$?&#JAV4B62#o;mxXUCHtsRr65rV1d zOzFFK{ckp~56@_aYUf%;!um0#A@u=&V_6RA9Xsz16xB}0A)XQ!V)b9T^(DRIFgvN z4$DNLkM*#T($^Bf_02d;iIpcL8QXP;Olc8r_26+Wp6%tpnvB|^+t-~cQ0J`oY zKJ?qoysV}SEi!k9a=_L)A};=_q|buLLBx7x(stI#fZTe%Mq)Mu*5?V}&v*<7^>Y~L zDDH*SsipDl$|z+Z&z0uq&3t`UzPiT)F22SQ)f zi~)ilUFX@p!HD>8Y)Olclk+UN@Kunj0&j**o9eME;U(FOv+?OPFRI8zY3^VGljU-$WnVWdTKl-ujg78{s zGQH|E;K0~Gicfz*Hb;Sq-Vk`&{%En(`dhoUKN>jV!p?{q5)6GMT}&55I#JWp2oh2} zwwdA1zAH&53z&M0dDx^;**&(qR$tQAPC=Mh@$uGQf2|!&X=`q727*wAoHDY~ie|Kj z_m^fifc(CKw{VB?sOFb@8mZKH%wy?C{|YKIbJ$s}LdYw8 z4w9!NMhyINBHOf@`g7d(@Hlefgc=@IE$9&Pa2nJrcQ`s^|Qt zf%7Z`xRjXx+l8)kOxW;>Q!YCF0BV#K$yE9faryAReQyo40on-5flO_m^Mw^NWjnMk_vis28Xa*Uhat_yD_VdDz*9$~%ep4<^j#&GzrN z(xRH5z50rU0DfZAeV@q~syBjtPoQlQIifc}dytgtJm0L@kv2j)i9J$30~58C<7NP_ zX2Si=OxT7@N?@HyN})k(4sw-bGh>jrLwecc|1OPcVsLN^o49K60yX70@H$B*8BDm} zyLSV-A+RSesEQvdVmeI=LMqh`CKxf&$$o6GRxqr30Li2UviXzk1ZwHGSEfpe%)uN^ zNy*L_9ubk2iCk!o*uR=ilR{$&)>Y3YXaP}fHo<@Y!sh3Vy{p<49A=0=U-g-X_$^ch z#&e?4KQM6MqVA8+Jt;jE#PXq!h^a710kp3`V<9_FNG4+#q3e9)Gtq1eji9}!G9Ng8 zyfHExxdd;Pl7A4d3t5E(ol+IvO*doR( zxKxgfyU!8LR#TIQkB=)U%C^c=B}kr2So$ZWvaFw^ZzjFIc!Zu+>Vp?FkcW`|Z*uIdJ7Rb+ z9NSq=5^U$XJfz(=ECwB@cN{3eO!nO8&!1aZeUTH%0)Re#qxLs%mVc+O=!<5KXUK%9 zcUT;cnr%dx}ZV=zgKaO|nO(rkqT&gaxF%mzP+4ew@1UKZ5BnCDG2$Kljw{rlIwdEkm8-=yZ`(3owSFF(*M$2UF2%Rx!7!) zA+7%kkH%5z*{wfDtN~2G@sKvVg~ZIH$y7##A+vvR?#d?_$Kk%_qhdb}q1i<)sD?q+eTGurA(rnKcV;xSaT5P+j=k&)@$@ z>GUb)+h>hE)jmjDJ%9Oq$=@EsE&RzJt>3zN*Ownxa{_~sdfp`OsP?YSli%<3qZ8g5Wt`fc4g)SE9#o@xb0C$E3*ZvWtO1^c&S!yIJ|GFm$#=?Y~N6!z->JH)p= z4m{&L1aTcUkG1{olPUF9Iz=V_-DrsF>G}MjWD^xP2!wzZg8l7ZdA;w8sa!(TmyCeNFjgbxl~dAhzd?L61F8#rn41R| z#_i5SNu_U{2h&4p+lO?EOZ-Qa=t&9lG>Oa8s;MVw8)(PM=de$#YiC+pKqFoT!n3`DjjQp ze~b2z{Y#DPhKvcpo0ge-eiVh6IN0GfiVC)DG+u4hpO!2GbyR^o&C?TE_ zz@Z-|9_9X?YyOtgl;Op1gGFb#Etd1ySZ%_00^0c(p`|-yc@W=2!h7%*#KCjX&p&9;D06=Dk`V>n?(zFZ2%Sv3fH6FT%AR+N+o z0uI6A6?G0v-n}+8R)24APpqe)=Gu-O4BxM%JsuDb2745eH3yFzLBv48Zc&i$=jTtJ z{F#vg{}emCt>!{5$H=DzU#JG_!FDEsTS}G*+|D))gl9kE)wNTX&P+yb&?3v_)0MQ~ zeczY!?Sedx$dIQ`BOwufOE$odz)0aQ1i?`tPUY_VnXo*VHTH2`Budt`JZcXY&Q_Ki zTh$wZj_m=tTSk0whQkJS=L_HHjn(J#v5(e>?BjXk=sxkFfpc`@k6K{f*LcqgoPr$c z10fr%4weiy4q9Wo^0)I(fX70ZANt|En8&;MPeJB5eDGi`7Vs4f4U<+xHi-*z&Rcxq zJmn^4wh}wy@Z>nv8>r zc=xWgykZ$qyzRyQXVzba1+S{BMSXf`EG!L;j=bzGjZe)(Z9gi{6+Q%UScs4o3xf35uUA)(CgK<|){3XuUvvwuKWbt{ zCag-b-zBKhVBNxeT>hwO>oec``l6O!pE$d`Y7^O32TE)9+4LP{LAQN1ws*2E|8L3v8T;@zqnQvXjs@`?2@M?|;sr2hz@bVv{N%l$ zhLcK#Xc*(;R+xOjpB-+F&t@4Q+hOK~U{d2wIgEa^XDyCpnISC)GBfLHaZVQ8;1I{ znr1K5{;01K!E)O^0Qd~lyY3}Rodn@wTc;U^3!3=5F7>dt@l@PQhCv*M-~mY$$h^RY z)_$^cG~UqJyht8Fb zvJVF++vo55MAv^X4&~*&EU))Gc>i}ye48>AV?qo3XVQG%u^+z)=bu)AjhHi#$pnS&;D!9Vj^bl6MOC3yAvbD z^w8_D8}odDmyCi!);A*)gOh@t6nmvyuc6w*MpnU6$CBCstVHteM~|jfrRiS3PL2wq z;IuHJ4RWV5!EAu9Mz(vB>ig8cr3>GBq)XxH_nUdW7YIKL;$?>W=#!1xxold=ArmNZ zHv%3ZzR@($l7F54^;Z6;E<0F1#(oIO5X{d|+!>#rt%w=%ACgW!S-zQ$5u5y3YWdpJ zJM@HxZj3Vj#|Z*JG9xtCzdNG}l6HCKd)bf3u~aYLER9{I!e%oMJUuUCbHn7_H6NdR zcIDajL5;ieyP{aDAtD(0>U!xWc;IqAFcmUw1};&Sl_cl&f_0IS?I+r?tTP?W;-q{o zRp{U9@3;ksj%+rufO|SXi?j` zN>I5CUOFw?EVxRQuAMJW`(9C}+grf!&=fT$dfWf-`#Rhf&OYMy#`!pH!^OA8uTw^# ztZQuh5UUhETHiuvQ+@mSFO-of?r*j_p79jN~I^ATiprI6Y)2vWsq3ZnSw{ zWXS(2U$W@??a0DAv_n*FFO~n=VY}{PjoFnWz78;VBzMxr2ZT zYHV&3_{)4KdO9ajq_)fz>^9+}_;x_g>nuZTP7 zihj9OLvcJZFkU7DWh?MBahk@t@q9KVeNtolHuRcB0 zwVRToa3TEc5g(tP6Xh-JH-}d!X=YzcVjT2N zvAm^eo@7C?DPA&^XMQ)ujx%=AqMo3Pf7d$&r9X|Dy&??+nwlNvicL;I_382d^Pk3} zf;Zhe3j#&Nk#kQYfWZVIj2UKw=3?Svxw~i4I|r$g@E%YVZp&>g}d6%%>gAncY#IqWMl-d{BAslJwAf;7_m)cs|C!b>Rp?!%5^e=_-p#7L6=tJ2mr0Hke-js%vNbQ-}GZW+|Wo5BN(E zwd5pJ!i~tS{6J*{-=Q8I1{h)XC)F>s3(WX;xcNMSO3d%S3Yd`jdj6!kLuS{ zKq_TnWfjCrxE5e)kq8UL7-6aSVN)S(R0s1p{UD{|pt<7i)@M?*G*1b}@3UQ0Z|>JP zQ%DU+Ja|FIB$+s@caH}ExX6UilL*G3lP2l~(t}2{MMm-V%tyGM{eNCsy@XKYCgdJO z6@p|Bzh4hH1JX%x`updHklhDFgV%&BRLZ|ZuR|G#p0_AEIYfo34Ju~nI!?;=4FSI! zWda$gm=B(35Iki`SCV`^#42c|s^j6-k!!_gBY++cfL#P}A47rDkIqs!)ZAwkg&`u( z{qqDR>>nnoA)63y`41D-0J=Oe>5@OUtsU3hN9IqwM$l;^wj^X1kj6am(*?x|YOc%? zDf>@XAvBK!r$7N+oq7;6iPQ1wjeGuW+_K2dJNy3qlWWdrhj`y^=s%3*rXaW^Qls_kwTlhoxTQ?20vGEs&Z< z>79`tDR87FcmTMcL=JBced>vk-J^r2qbATgIuq&69F~u-^CIAs}(QIKa4jq%e(FgXzOrG1*j1N04oVQ&E0z*7PYHXSHbOf{#5H(!vY=coPVK6^4GMh-kkbVh|0IFQBwcp z;9QPdvrbw(GBhx-bMRLj$^SE042J3)ngAnjpr#1dh-|7ljo#l+jj>-~SF}ZO_oM@L zLFFa4u^q}>Ve>h2%^4MC{?s$+v4~{g{9T?=NsjypPFJM`BPD+@Eq_Rl*e5RG4)W`T!1vHL0u zir~Pb=PF6DdHvFZwc2HTmDW}OC%V+tOUY^J_~A8|d$O5!RsPT1)A1^P6tc(Oug`1S;*^ZAwvJ=nU|Y%THrgVr%pac5Z|Dv#ecH;c3p#cppb0HE z?j3mPF}FYSwv(P@gsGC!|L7pl)?uxy+zW0XMS;|OQ4o=irEQlGyf=vbzi5{4V7|I* z|HUuwX>i1h23w9$%13(Z1ER^fi~8b7?E<^&Jzv$1TwNNUQ*b;W=wjY6O|i@sE516< z0j*{t_qPR+Y+RfHFojU~2?uM>-o5b1bKu*)T}n=ZqH+#)xWOBugqGb=#NpX{tjgR^ zMbYuUkMI|8#g7p6Uk}LS>L~Mw#UUaLIW)rlA#j!hA~tJZ;@`&wY=#%E@5cyu zdQp@jxmBn=u;~Bz`ZrBBcQk&b0Zu1rv=NHzIZ8NYivkKtd%xe5p7P7MrvcVVFyT+Bl zPCYqL&Zewh?5#90F==+zD(KGV&|qH8#)ci)5r?rM=hN!eDBTn6CxF0`-Foxd1nZsW z$zex?oY+PTI%(%9J<&)2Hpk8-XxW{00UY!$sPaN=@)Xh9cXH;(FVD3NXyI61OjW!; zcv|rLL57*67`SZ*h&M~_F0jkmB4olJH=o5&?adEPfF@^;h3oRq8`&+lR;*M%V1A9a z_rigj6^RzVbN=4(W@peqv0 zRA-;XC5m+cIt}Vo^Ox@YDfq|)B1R@S#Kc=0?5(s`u-SlxJS9f1P^%0N_dN=AMz05M z4~{>+o&)is2VntRa!n#>nwmm1k3sY%q$BiRtS%N1aSF5xgsB0jg_mTL~?MsajFC|QUi2m2Xgw%RH(iX7Dy@tfEX|MFPNNF#7QD!J!~MixM30}6h2|AV#LReI8^W&6`Mpan1r zz~Z#O%-911w_ZN5TQ1jQb6Zb(qQ(**4Fxe19?O#z3DzhK-M&0DC8m33IFPj%;#Th? zAgXZtyun-Q$E1s;okxaoer(EtmbhE<$zW?~%yQ$XfVEM2tnaj+*KbbAq7O|(0` z7Znr~`tKM2j3IVpr0+va8Gvq(7Y{dcWu%Ij%K>Kw0u*O@!m2`x3T*>{o{?q6K6|!{ z2#Fhr8dK)A#%sz4KB3yYfmi1$3|K4EkxMu^GL+QZke|j#4Hamh&w3>=%(b<|2m)JXX!~E+7E|(B$ffZ$m*gSi?F9NC97I=f*XH<%O`kI z#tghH@v;r?fQ!cYD^pSNNn6Zg(zHi%H9b9jYm%2i>A6y2#Uh)|HIi5>nUHgWu3*a?xOwbHI?3@lC zb3Kr{P;^%sib^?O-YX|37Xma~#&}gp*uh0p;3~XH(?4wxGu+pzb%*sHyyNV8S9RgE zAyO@}2io9}0iyZ%jFrXAantk>(8;rS&B0|Rw*6u2>xqjnR7htsAq6m-sXyD2fnrDFc_+qT%K3%J(7G@|NKR|<15_cU;p=;%ww#& zgkW^>;>ApOr;||Y`n1nX9i=?1uQt(3uFzzRrqB?JsRtBFqw`P0#> z_ejm4$RoG$61F(L0c`VPuR^Gd(9jXkPO);{1h|u^8;HazaSWMtKa4$gOUL5J#I27S zOW}01YJCzFC1bIwg!X@=O;~vSbjm8ECl)c2UhB>dG6Dko1OfG1Oe67_4jA{GP)pm5FRwc2(RX0&M8K4czHsKtDeuRD(b+?SM5_P*@-;G0hkj z0@p_Ju3^XghyQXvxZr`@54`#MgRD(h3~O0G-h=B$DwR3&e!NQu5aABKEwT5uvet2j4YICwNV`&`^+QnN zX$pM`hV;p*m27Vu@8$?N$#-N>tI)u}zRr*)d*b>A!ME9G14900aLp3i1{_=u=7MyJrP+Xxmk7+w$6q(s zA!cpV!VWfYZGx|+@ouh?j8WW*^pW|PTz1}9p(exGguG>}kqm){t0GK~D_MZX5;9kC zJ${R%NHbBZw_!(L^g;>CV+5L#mSa6qXxG50TV#Y0FE&Olv`YuC zVb4R}^oapdb3J_bMJt?@Y*IS7RTTL}h^wL(79`O^Z|ci;=&Go?y13*Jo{m7IVRV}6 zI>mrnm5GkA(Rx#VPUg%34;i14LQ>9NIQHip5=zKs3v%>z&Vzn54;+3y%=(A(UDv!t zaSdujcD48VlA1%PmFMX!-rF6HADp<(Wn=gH8O4pTQ&y~Aq2{XT$J^^-K&rTP9o_wN zd8C~%3RpWSQnp=O|HX1oh?D5}e+(1yIHh0aE;m*@Uj1Y?>Gh7YKVH7l9g}?;28|#D zB?DgosJgn|omuL(a}IfTk;Q7_aqEd>+%lHQ$CnW^eLcZ5FavvQmkC^ zW5Pmy20?N-fwPO_Ufvx1m(f69Q}NCKmWuFR%9UD0(Z4ajtwWS7`9;p89S(2=JBLu< z1ELRHSqf9o1Egt&&X0=99TZJ`ibNiI{``5urGiYYtNU{CF>UZ%iO>~|z507IODHO! z92tgtX16n&2^+iF?dOVEiThZs#B-d`wIHby0Fi;{l+o0(su$V1^^mIcx2qUxlDKiN z36TNb=e2IlP$bM+2)X$cp`#XAZXB`9eE*?|U2~RI(WnavEJ! z5JChLi9dCwEix!xqWK^`c`=@_T?;NlJ51%q8t?p(mF$`>$0CfS7>;)nEaUI7G51`@ z4t$6#lV6RjNQbVA2|BI?muJSsG>y&|%^df*=)fso4UKD1@Q*1$js1w z|BZW(8p5aFI3KY*+++CWp)xV4eYKl7IGSU=EwC#F)`@jH2btr3b%xL5fPqHZws|*9 zXz~WqDg~*8&M(a^W^wBNtEDA-SKaar_8%WtRaS6N0o$Q`9+-NFMWL>)KG`L5c?a@2 zpGjhrO!}ml%pqhAsT5JL6A)AE;zs#u?AfT#@A)e)X)1@QtPjc&`GffkD5rBWK3{XZ zr1cGkk=x-RdJql!y#q$zvx9z4bLsT!Yj4kc+y_aTuJ+aCC4O~ljDqf7Y-RG!mQVeu zyPvfHC@CGo2rB3%^@C$ikV&>p5)rT>0SgI!RPqR7oDgT zo$&oCHImx-`1{?tGx+it#U-ui+?GD-_S4sl?*&AE*vpsjG)4W`S?{@4cn=NBjR-j< zJqCc6=I>fVc`PgxUtC0UZa{~D5mcqhHJ_-Ewkh`xR4R9dKMKwrQaM}zWw>`Fga4T+ z#b(j}!U&W#M~%a*Cu?Vp-?3}x_`#n@2Sj3C`Iiw2^4iAqvi{g>8eRRJ^w=KOPr7Z# zVb}c@Zji+N4u>2uYCw{q6hM5~;e*Ffza_1(_K>_fG>xYsX+um*>blqb+ zS<5-+A1nxn?JkCRBF!Q2ERh4Cv802{6=h=>&nAlhx8bso4D;%{S`4dM4ztY}>Cz+W ztURZr(}I8%#Az_ED_HlGo?a3$rNFiwWLxR&i!11PmzpKnFzzR&8y`QL93SMquVy0- zPK)A6><={ZyS|uFaw<}`w(R6jwtH#|1me=>@_DMV*%y!{q1QRhZAHqUL$#b;7DY_CC)Y}-6F`<=9Gq}85>WE=np zk|ozz4C?IZ19cg-@j6*&6XqG{-FgO2k}XSK;2_{zxw2<(>LCE z@z{#Uj0Ez%RblZ??RH}^lZ_tYSx{I=X83&md_fuWwJ^0p_{J*lxNo3R3;CLwwfr65 z>cp#b`DgV`<1op5dDBBifwI&z)#kTh9bfoa@6L?_w?72K3hl>i^6VY`oyp(2S2w)7 zuTsBA#Otj2WlOraP}s#tXmPosSPP~{Upfy0SkZi!m%m(Fy5uy$AZn{d1n0tJ4TC`6 zLW>a>zcuXEvbIg8eR+I#cFFEQ;%-LFp>SSkz+xP!KT&MFe}5VrJiRSjvR2wJ=%J-N z5>uIBc%&OMDy^4g6j&Ag$yl5}+eY?oD-~Ma1z?(3PU>Ko6DKF9)fndIn4zZB$#aZH znz-sHrG|^I?5O!q*YZ}kuQ|N&MUm<}R;}?N>rXh@&R$F5)k-A;SX2-la08j)@BpO_ znna$^ry_&!fq!@Jr(YJhq$Qg8Ww&~n(l0>`G{;Q$ZhO_Wyo)G_Na}XT6D{Nv0?w5G zwxCDFM1R*|&*~lacQ3}C4jDI{O9^o0$PTwE+%dz=(tJVoqJAE zxCd@MG#afd<1!Wxr?ov3(2u)yeBIiOU*@1>H?r32>+6O5H8eB`Bn#F4`;mO8TZp@W zuF1A=_f3?4k6g3tdwPn96#_t>=%mb#N8oJXK3< zt4lj6@@e{^L--9+3eNR`+?L^6A(a*MU%O_F4xNjK$JP}#we*Y!6h~E!qcQwhL*snH zRY5_Poi;`9ShwDB0&x+MNtZhu);oxa~ z9W4LgG88YTAXPtCj5i6qye@6?#*Mur;7!OW#fFKHmi=q-q3TmjJC1*5eUpT*5=R%t zmT5ru#A}YGJvsM4pN#z<7y)spYh3jOW}4d9mq&!$Si#;n@VvYC`_PzfYnws7esSKH zKldGFQBG?v=87_>$|dXPH>G)kO-!6-t+|P>hw1et5nTBeO%jP$hkZgW^q8gzaQ?jp}+s;J!k_8KHvC|Ws7k9)291Ww)rLX7zKy1BGG-p z1c$EH4qj(3IHJxDSX6tFUJID|rCYdL3e7*LE0t?2AWtU?oD8beVvs&9A+t6f^g=yQ z8&o?yK2`L0HO^tl0MCTEa=7Lu!6xrznE;#NGbDdI;h8yf@_C3b-0zhQP5+i+a{* zzvDE$r#Af~gf{uA@#4DQJBH?eVFP{vK0L0KodCSc@5L>xDfQd%dursO$^M#~9*QT8 zmBVCP3beJWHZ{MV@q;o_oN9L&WSGBRYdv^!*v@h#d_2iRB;CtUQ@b@(#*%7Fq9jswhZ?W=+_u|sOK}MVaa3Xp4 zu{xmKBY?G!;5@$0fL4PLD~XF5q0f@0JFwmnN9}5ETSYX~rTVj&NgVx1CJ*ytz3=qg z;}ap?P<9>iKyCO%L{dvT4=X4T+hlOz`Rr%CGS2Y*jX^>}omTBd`lu%ZqG8P@Gxpwy z4Bv!;zahqmQurdH*8G3h`V_SVM)n%jeVC>E%8xfBuBjk8&rhE|k#PdhP4=R_(SM3; zfk;|<3S6I!z$$u8+P#T{{tq-CDewn9km9Gp;b6h12>tOSLWR>iv1Z5K4f3`mVc>b2 zBVXr8%CMexrE>WRSTwf!Iw~p zG;Cm4^s=;cH3Q5ha&9IN)6d|;&q!@xiT_Jfr7_l^KjeW2k5*G7*p7we>-GxQtmLs# z3v3ZoY|YuOzkTKL-~YGPY2Fr$qy(**)O2Ke=Jl1GE$R6G+mANxCNvI^!mL_B%#DZC zTMOL=Ejo4}IQ(fAkY!}e(i*CsjC(=jn0vqT)1t^1%C)R@=V&M#7FmFyfUuyexEJTR0w6D$sEYwnfMD@I zbzqL*5H{}B!1g1y)}+M)9wl$N#czcsutH$V#FRqkmJP}K34z?RCja5gOx)S+f|Vyn`m_-q z!mZ`zbh8m*sI7#dLYIahfH?pcTslxBN*6zkBZJoflrk;-Nr-DE=CX-LmZEWm`J0T1 ziIz>K8Gl5o>wbT(zx$}=?Y`ZMx-HN;$k!K;yK}Rii)$>Q=;J`)^O=8lOf2@XJgb)x zvb8@quRn8PbLaojA|V}<8-ub}Xo#*L!70%{qp3?=j#H!y>+#mK;T@=rgdv9bmJUmR zYy1UuzlBfh!9$1g!5-fS0b){a=k?zDckV*gLrLu^hK5q@ezWzGqlmD6kS&HM1ebw$ zYS$I7|GKP9QsJ_VauAY^km>Gip&?;Xf%#pylchDf`eK%$jbq3b;$u^~g@0w0A~lV8 zR0hw^{?%oV$sGc!^fjb8iS~x3F0<)`RLja_1CNYbgGnd`Y9-PmBK}a?lS7WWL`W|Q ztiFcB_@3D0&-^Ky+q2=&9pTUwGXjV!7H=;UH`*vH1ytM3$z)$L__YX}O5(2_9KikV zs2``X>j1`}fOB<*-kxR*-s&wuDxzI$for;E_zsQO( zy@+T34<^^Z_cghX2dM&q3iW02z=cAkb!I%iD=EeM7Cy|Cbo^8G(6WE$shHK>L05;N zAlTf8!76c}+r5e`Le@EO;#fd5oQCzt%+LJdq@=eyxv^J6u0C@VKi8&HrxyU!5;4b_ zwBZwEkVz3Nj@xrY9p^bPT1?9X1hVmtec5Jb3FB1}*?>=L_|_`)6|R<9l?!}_~0Va zfe=b~eZ(<$3(H}=>P9;kT&IXvCjLDcxEp+AMJY=T=ksac7B;=vX@}73?6>QujZ8Ue zlTwnK%i0HO>-KZ}_h6e4pJ3i&fB$wpW&^=-CROP*U=8^D1yIW|HoCCtH1twfj>v> zgM^N*wUSR$LbASunohD)L{sf_#vPp{+DBeoJFziM;XUs;KKb=6Qn|SkQ-A4QylDG! z^@qD9{Sz)>V7R@teT zV_!cu1FxV@Bc~~L_d~Y%*9A{0SW$VBKRls|a0x`+<*{R}AyI|bi%VST0qTo>Z`tua zD=bVqdbK;-#I8SC&vY!2wk=3#y>IFviXHdYy&+Dtt1AauWp$pnD9pyD3|qQ*Bq0(S z`cOKg?MtKY?%7b(ZjfsoKI~iRF_$z-b=7iK01choNGFzw#8{aMM=l$`m6S)f{QVlP z9h$VD-#MXp@=B)qN-gU~k(7s@sU@sSY~AS31ZOTCWjtER&hD%rL8*DNF;K$_MbG4q zkmSi;O^$M_Jrlcj&R%m24fp~)qRX4NP$bIyc!Hya)z4+o3!dyrR;m1YZWd#stL8K5 zH8>I8HJ`%|KD>QYU(P;Sf4-o6Z<)Q^81)k*PEg#OfyMcVlLVz|QL}P^{I<^V#P6Q@ z0vZb6eLZHWY4VIuxJxoDnKlNOf9j{ASfFW=PJO)TEm0pAc!#~eq4LP%z-1v?fp$Q{ zd;TfguJtr2qXhq1fsGlD^rPp6c;;#yFG8W(Zu3aG!p6N8&8+6834WGx-+PSwn#Ar5Rp-(E-%P1Q zNuk~EJh!+g?W9SrvSHOypEDIaDepf=na#fNW!M`ve+!hcX+qO!?Oy$H6J6Hvt z{>OSir)CF^sd>lxZMJA0`)#(a_5Qwr{EJt9?h2s$)t2Hv*?zpkT5?`ZrYcWRc< zYAw$+ixBo%yxJgd^iBmqC z*@qh5`9-+dAWW+{e|%hyBOJt_eM6j-sz>21(f!->&Nh#ITydPmc=P`5qm5=2A`Sf# z@)`1DA3jYga=bCy^*&46T2&j0?)m%RPTdydvzjXw)Mcdr1WlPU$y&B;GL+EFl-l3& z%-#b{O7io=`5+`ao7t4O7l6|W@dg>*17I&6=^Qc3$Dp)ZZX-RB;3#i=^mfvcG+lk~ zE|jmd$&)zJZl#)SEpj?jb8nn#Q24`FN0^9dN$xv5YSi`Pm#F*q?|X0+o}H54xIw~Y->h+Bh_hqltZATz1n9s^>n#|$1De#2suGI)VBt@?|iO%n~hs; zi{j4Yq^1C6cgMz@bF#Az3MG5_aj~Xv&hF}YT^@zQU+=(d=ge+!7-Rvk;SueLCmg($ zJU6pZGRieM*fb($#^uJ2Rle17yOc8j(l({yWF`X|bTG6u%eFlQMxDA0#4KJ#Fg5~S zoxjVg?>XAIC&Y98PD%#9An&pJacs3X0bB63&(E+=_w3}E1=u<^%9hu8K|$+=)@dVT z&ga8H#}hzDGeGphn_o#)rTcM#d~^T94KOlj{ey}>%0q{$w`q(p+|36Ij7Q^V=v zcw_X3*iFhDRFSIj7sJX>LyV};hMC775yC-5(KIx)r2@QEgFrEC%Twq< zik*1MhP{U)m5=L1n|7Ob{}naAV5bTmskP;PXM}ri-?ohm&vpBJS0Qw4^~#m$H0a}5 zzx?;YpH7X}O391)FDkoS$14VjDT_)HOmy{h2dMEH+o^P%ZvL;APzLRc($hE$Z9gQ} z-P$>Kly`Vxd&4o0YBSzbTiR6wViLFSa>|DoTMtwH8j%-S`ntBYFEh>Oudfi-#|9f@ z8p<`ZOk0earC)+=QrY>16z4qqv?;+Ze=V2_vj8Hys;?FY_wU?UaftzAHS^u&=QDLi z(QR+ZGBPrf;pU>`lm@9UT1T=FPu^^`kTLGzxllc5H1;J7sbIDE_aAVr*92=5$|S31 zQ()+mll`gpg2$$+{|yWXDA>4-61w}cTzX2b+10km^U2Bd8QH0$U7BJ9jQkqCc;cb5 z>}?j1)$!IFgWGJBDwV=lPuPQP9pZ-= zmEyaVSW(`Q-yk{C<2ahogg1|ovSvhGQeF<>;5{&e8!;Z7G+;n#N*{%FrW(Y$A_tdv zi?=+!o4ht=o$#oPx=tj08$Fr zjAELcl-3$G#cDTslqd3u5SlGyIi+3GfRhMY4X209hIN!2)zioRQJA$Z`VoLnU+;Sa zgQOZUSXh;s4Gt&4Ho#;n_;n*Cr|fGq_pRm5$%j?F!_O>o&zcrsIc#{3{oC(5jG~H+ z&{a1vG4b`puMopoGEdI;?t+q*?|Vb=CWGpogHB;ti#(o49llj5y=fgQiF9|tbVx;f zz$CGlz)(*g&z6=Ja-AsBJn+?_g;H^T%|}WD^!oDqJ9g8=88aC&rQb$biz}iK5qm#> z4?`ys^(vWUOUznzsQ{apNFn%^m&(YBw%on2q-A(-i0JrZUB~g!z|9qghp%0Gs(a$9 zzV5j*F7PD%u};>u zPota_o{X_Zj2bsxg3;5h&^vwn=R^^R4UQ<8@19s{SCu#Rh?W(r>4v+TMD_|lLv2d%OBEASjCtrDXmkXFrUsJYt=2^By$u%Pr zq_=**RpHp(Y1@PN@PmI=7OWr(oc1JSk=!0%4_v$MH4|uag(4N zeR<{^oBqC!3(-1r^X2v!HIk9K{9JJLYN1W6HP|7;9Uu9#-*Be2t7yINt4mZf(Y|_B zxyHC~ufCcg=zkoPtSB|y6O4zWZ^E<_FINk@xwUDhiJrD*Oy|3V?YZ-%cc?fIMpiG9 zZ?j6gjx*ef{FsRtR%?%uBBmaK6i3IUR}Snm+@s@}@7%OV#53a4@J?#cXg+)2GXKKZ zZm)~J0dD_J>^jbT#CEdZ&wy9peGw7goIC;V$NKbE|MkE3D_ow+4z_9SjC#ZT!IFtP zNZd0H=|^oHd7UKX^yxV@{kcr(g0)o!-QBu&C%w9&v9mbGAnfl&o*Vd>Um_9i!W;RsK||H`{Ar5B z5Xyk5o?9y$vlUFIdc%3*lhR&i*=p+K*@S<8^tgCjYqXEEvy|1ZE}8fnytSGIdJ;LQ zZDCUvM{oLPTFQm$KIN%$RZh%`W6ZutJ6YoGv-z-pmAV@vGQ9WG-6?vjK!T;y(8}*4 zLq`%D-bV|vGI8(x3F$~ZjJQMVf@lT7d;oy-)d|U*&9%i#dLHJR_N4T0qh{_CKQ1z5 zo9i_*?30iXKdSD?yp;~U?z@~7DenP)5hrjmLBr-~*o(7jP_8-xB{##2mZBBxlvbgr z+Na@$d)24bd>ZMP@l2|CBuOob2^sG>Hn?2o@=!8L;gNGev!D5SbYq29q9&^5yvG>A z4Vz`jpVIkAw{UgDKbhDMS6F1aNgby2{JPDrCv`@4kH&I)^-J`P3gAITrCO9ez9M^z zwSz+w3m&7d*6E*V)w1XQq%eqQ#I3<+>q7540?&1m+FUQhzFtkYY&Ot2lbnykulMYxwG;(ZWPDTWmVBSl;-9VAddiKvBQ=LWBk>Rq+(buMyR`k_P zM`iY2em-X0)fdTQ7$bjKj^`-L$)josA|_!w3jZab2%69FW5?8B;E6^8PT3f6v8dCe z?pG!^4% z{tG%joZZ7Y8p0LFzdzE~WQL*9*w9EF^UM-gUgpoylk0AyrFSZaJ!&!Y!E6pgc!GIz zCXIL$5#cS6Fo_Xr?!`C*aQ<*jiUBZ6=?rGTJzmZ;uHFT+KvbnF^U$0EgH#Rr7UuMC z>TXJ?yD*o`>B78!d!{cJ#C2{zZbb*rvDwoTEBr(rC+{We`IjrA*W14w!S+;kBRI=Fcavx%Z%FIcd8Qi1_=lVSZ4#= z{9w@@TmB`sEX)t5nU2Z#adubaV%Cz=P@1Icen`;Gwt;zD{sqy;Z!2e@YiU4(f(%4b zB+i~>s2Z6Bm5=iV}}r@2F($*zb(tteQhM&-iI+wX3g`sSnkrh1?;o0T`z zyl{28?ofZh28n5+Ck88pa$|IeZ6c5{^da7*SOc}r;!WY*=y%lTOf_tZ)sR}`b1o{G zcwx1tmt8LB;6<-X$v@w1%l9NeSh(D1)(Rz8edbf8qXI7Le;aG}P}Z40lvWYZ73Of&m3vdp{3QLFOc5I43a&$;xx>+O>rnV>OVn zvsr_~z-En1PKcXo@S0*by)odu{&$lZz0%3CXO`6>!p&j+#gWxbu@kfBT+LmVOqVaD zmWPee`2164G894=9|wN+Gp)L3l6Z7 zb=)0m5hF^;4*e&kqpy>~dqix7>$>|dy>GSb>lt!Sh*s|MF9JKbY$75TC+ z={jrR@sbDCq7%G!CU;&HUk(*fvDZ}4x)y{1hyV==jzA?|U z%2G1);!A~E)+sag5~qwQJcaXRY}*Rz$1bDdwaNy;@nX~X4EV`Jsh5S3A?DfOtr`-I zo;CGol==*8cdVFV51676C5+P;wzl@CMCHL?&npvH#R7c zJvBbnDPzq2E<2N_N~nRoyC8PEmwn;3RjOm>t}+*6$^BL*^|we1s65VS%#&#obXA<& zG!kg?r*Ocq8GX1|EkhWp`nn)P*Ygg-scJ~{&e)!%6p8x6i$QH(J#mlrSa|EZL)Sa# zd9KJnH|IE%;^&!P0MM)2fFCV$wX@h_aY)31Eb+SFM9MlSXP5|fKH5e zZD<|cK&c7nD51!Bj<1?>ueB&lJDaOg$V;-5%LRIjbO*6+(`J5&XH$ZYS)D@ZRN(kw z@7QE@$*4=A-o9H+Yo3D=47&9_UT_YM1z-wCYfu4MXx@8~Qv&W%E9SO;G#3w!>xzT|rSX%}9liiWpN_g9V!X?^-y01lvDaxq7vX z>1&Gjmu+{BHA(ruZSb6NQB`nQY0b(}j8gmJ^KI%L9Psqmu!Z?JzODz@Z-_{>x3yUh z`McE11(eXpHrn~os;zs)`e&x&uxPTz(keJHvel!kx&s;nmdd(>uT~d7D z@&NfSaZSZiC8^`oK%ehGYqFmo!M#Uo5?7wm=yp59`o5ZPjyJAiQdFSl=7kjbfO4~j zRNI`xs_mYE3#XRX)-wqPQLuSj+922&ZjpUxF^5zREDVF$l=3Jfz4y@ zkDd0wb8k*n;3$-0bWe5)@F00Rlr{Dw?fBHs_?e!n_^nR8Z@lKPi<}Sz%&J$?6jkQe zc)gG!S#+c3Y~jy*p1uk8;GVEo{nqs0rHKX%i$J!nb^W%^I0aOik6D5yN*egjr)M+yTb1Et5t4~&u*L!tIYl*z;!oQMznoou5ccbVM-6?pf%_M#~YOYwt5{Ke!*0a~TjrF)})PBzK%3-3yn78(uPEL$%?SXNPPq(aThWdde<) z>ig~+9G*M6lZo0%T3C3ep6$;G6@90VYWZW2^4zAr-SZ#u_YY2ZVKKixGztC1;J2Qk zI6PGq_783kPaM5ME7~Wk8|4l|$+QX!o+B(gU60J~>JY!IwKu69Y6f|9v&u^ySk1pHLVTl=ovS8}*e$~`t9&pFFdEbGw*fZc?;lk%=Xe_wC&K0U!hfoTXGsjM#BS`tW5T$dtzRUrU5AGocuPhuG6QZVR%T4z8S5%1%&(=>7 z@s~o>gm?bwN}+TIE-z*DhAEgNXx}%>AA1`$G5NwM)q<9?XugsmO*SrZD~TviaB&Iw z*P04XFEL?^AZkdoygki$b3*3FWgbfukI98eX-A#q7;N9v&F?N1{sEym$g#3i@sYt zyuH-A`i&My=PwWbEh@d^fR~A|Jo6yzP~=wT&sREl@NcwADK2+w%^GO?^pZYb6?Xg4 zzVZ~1Q=~e1n_HiCM{p)-t-G(?(rWa7K#9`Pv?ethGPtgYZ2_fYk;ciTeRLnM*um#_ zFVVLj7Z?aAk83qJ^Xy=;=QkNZDFkOBf4coHg{h5Umiy{`A0i-Mo|OEZ;_c5ndZRu+ z-E^mu+7qQI?KtC}WRa%OqOHu|Fx%@7@g!!#(#Al0YGQRNK^P(f)PTLX3(8XD%YNnN zOzOUT(O0;r+g!7`F+6JqU}EnbfgPUAjq9E7DjNBr;M9@&t9y(EDHQAHz`^bZl-HFH zb5QrD1?`E*_F=WjI(~j?JTgIqr|;kClg^7)YXij)vJa+n+TFB&3bydAg^*as!9DbUlaYn2^PfAUMJUeR}l}!A#Lzm*s zcFyG_`>U%ySra(}1%Jwn4r=F(Oy^EAqzp-Q&3>4<V$JMHxt*qGi{)H#M5A)vTxm?7E z(!85_Lq>z4;s#Re+>L!1x7RcDL~=9DXP6ncpL676n$rqrIsbfpXk zjdisBxl;a#pr~GF9Q;S^g6947j&{4}z4NuA)|M29;gJU$$ z(pQsxEesOc695*elu2RdZ*QMCdB1HY>9O0ml5y8Mx6X6Sr&x0iIW*ub(@K2g+upTK zW%oKWWwt7Tb^0F?e#vvu&nZ#QpLRIXA7q)DSWXA5*2!}9=S&*w+O^ZNPt>-2={v)` z1D!tGdYg8{bMEE8>%fwUigsd5x}Kh%FC$4Sk&L6oWHw7*KfkU@Y2>ML>54lRx#c%B zpWS74_~hS5dHMKLpPu3oY)_g={*V-K%Oc+k2S;DVDzxL(QiW^tjE^UzaPPZ4`{5pC ziOY?(*EfL_SpVsjZzlZjFpx@_%o_npUHI!}N{7d<9P3itZ3+$}O{vl5x@3+kZwIypfOJZR zYgw3?eRVpG^+wnrDTQbrxDRj;pwYsjLLR%`$7@|`DTXrN4QXFHDkslQY=be#41jCq5uOznw%`b?kPc6OjV(bW`fv!PCmJg5|#Sl^b(Iuju(^R6DQbJ2ndmi03 zUiap+fuz@y!$5qa{fk=?U@FRd_Z8NROb4nL-zDEhYYh!C`pZZd$c(FhX0Bak@2a=A z*VIwZpIhd`CO|*d^+MYYj#oWMAmAC=aq+;b`8&;u4i#T1TRGA^;}OdttCxUwazu!T zqPmAwH2a^-@KrL^YFzI7MMbD)8O1vla@hXKn_CHoCBwD-99h}|*@MpfKI#UVQQ5_l zT0?>caW}UHI8hr;LA&P5T94jr$+Yc3W$^w2XLC{#aM}Q{%9vF>$w0TAk|Iv?cxbal z6(qb)3D114E|4`?e=C{MknL_tljPIXzO(t@ekKAZ@AJ9wYMMaiKg!zz5pNn2wMEF- zSFo|Z9Ryksn)~DZG1}$tF_T@Hfg^|%kBTo`VcR5N*K^-h)bO&azujVZ$&^z+Zt}#S;_TJsHxDWqz>2RH_vXj_;A!omwdCH0M5s^6i@VjQlke5^ z`X~wBm01ItC=K$QuxmF+9fn1y?5$1Y&3QM46)nZ$lQ8=B{O%-O=)QUR_u~3Paz)2) z($0sCDd7+|vK4MH##)ub>k56jW@st1Jn>%XA&${6-#Urxf;bDqD}adqKuf@AYM4St z2z18lt`Zc}o(YGXpr{p9El%TnJu2aS;GXLJou)iA~EWB@=&a5!NX#raRD_XcvTZ zLSiSB_VpaIi9A&7so)^S7*!=r14nSk@_Il0{_4o}3D$X7GC>X=5Xq@@9#r~dDA1-! zOfm0)XNV^BwIk>OOEEv6mzQ5Y`^3iNjU4`UleA@Gh+xSrcR$vo!mDb=0K*|YpKIN} zy(vOBF4p37JNF)m#Mz^bY@_|c!YXL;F}<#`TAjhZ&${XWF%W44qr;Ds4m7pCms>oY zPzbl_W+|@Jwk5@}zMRq(O%w}FpZxO0o@r~-B*irE`*90imP;+_VwNLS>INYDe#o)U zqQ>bSeq)i@Y6G!Yo6+tC*l{BRYog?)J5~sJ4Q5YvvZtsW)C;Qja%b!A`PeL5+g0qY zlIl^QH}T5Y;Lh*lP1>c9T41nzCrcX=qm_9~~@x~L!Zgk6}$@OklHdBirN4VY+d#11G3q>wL)`_(S-ZwD@scdg=4>`AxQWLqs@2DToqv&sk0v#o~@oe(c zS2o*d-wSluV3^z&f>FtXBB)C8lu0}S+yM&hWc%pCigWmbP-3|CMX%<62% zBRk!`HZFO@Xlu?pha*DNHR$|S$=P)zukZr83cs~j;KHM^-z|=X56a$PWYLwny}x*{6m?~lbzUD_-iyUcLvwD!+I*Qx%%dy?uW2fL0jLoxlH zcHCyCp(<6g1(dIuq=WJ`?RlP)MMBfyo0Y7XG8k?W=-z?_)0E+BonP3yrxDJ^qbszq3dov$I6~ zo!>8w%7GeWXG5Ga>QzE6fD#Tu2(zZ87P^fNf6>QQ;kbHjaWBCLt?%Ed?aOd{XP9-#dB*R>(G4*$REU5ecW+?x5}%gUU!h7T3K zkF?eKO%6~+8yn6HOrnY!X=xQD4IaSjHONMszP(w1hP=n``^hzy1hzVr`vU(X#VjBt z+i?jQ$Fjz}?$=5q<$cy}w!KHYeh-!$Q3BHmKnT{F`RiBPE6<^c|ATdSdBQRu{bky>>)A!P5>c`@qr}KCA3EpbwS=z=GorfOtUvC{^LYC!Or+}%uq7-H7cVC3 zEl{LKm9^n|N~A$20;`e5!39uiMcua6PDlWuD}1%Q=5l-bo9hYdggs6ipkVv;dOZV50tm* z@3Dvw-YBL9v>6#tRMF||Lz$y@x^~H)DO3(mYKvX4Jj}`Xc$(YxZ50EbwG~I#`(~M5 zNK8u8Ik)8i=i?yx7G_rT^)B08rr|mBnOipiS9>>I?g+B&t@R%1nk1|F@MMpVmwWnf z<~{+#p_F6w@9y7ja#|~*cG&D$pAcqoeHSA+IhZ@oM*WwwH=;ar?9TYc8%(xeN#!gA zNQ^b+Ai$E1x5(|+zwRHaEVJlpntD{+;JW`dzNhf($CG6L=1Ds}OZ)DdAtqB9*?zNb z1yI{-m$;=9P~aUr$)xBMyop-gg}z>^busj~LW7pXAP6@4o|7YF+D+LfY>_r2@rR;( z#u2g6s~U>7?#Bu&ch~^DEZ-}0biHSu3B$ZhH!L_ZyXbTj2p?|z%!)$(BH)RABAT_L zkta4{LQ|C?)>dPbT;Wk<)lZtI$1{ss15%z0_lf&|pH5TG>pXtk5I0Kk&IY;Z6%5x1 zRS>qFz5~BM>3_YtsnKZZ&iuDe5WdYq0{3tT?f;J%bcKZstB@cCRrLtTOTF1%TJ7np ztG(jb+jC&D8d>OvWlDOoW3~Ha8)J4{5jC;nSv^EOv|!qc`!+MiRoT+rz(^*|ulM=7 zA(K1;{Iwy5NsahRGotv+*2&R^2M2C)?tSVH3@o_TY(y=ZTrQI+I|0dqT&GbuhW_q5 z3UH=1@uQ;5?@W5?I&0ai*eWJH6^K$mF#k@$!|{vOs&0D##-`qd2aH9`b@}}U`npho z0r1RD*GblaVOQl8SZbK>(_j$};D7Uehr{SIG5gtkyu5F$E=|STmmw--aCn)dlf(>X zm0e`2;%M;o_2V3fxs{_*qg|6>0nz{3>^{D^LBVJlp9QGEAQ$8a*^o<@;Q7biYLV}& zCm73E-Zc$zqUG*LJ&=RRY#1_h1|%(75#*G_uo&I#DDcV~KK&TnfB-huQ)S+n`MpCO z1-04pYLJpf8d7cQiU#zO+nwb5op~Bkt|&B~wYK{9fE0`!-`0}q;_bGm>tnqsyRS@| zy^T1$dVr>E5?H>Lor#lFZR8E|`~%WnVh;mtZ|EQ#4eQ|GO?oRtl?uh>=o@mw+Gyx~ zX}jnPww+&_VODv{&m$*Xp4t@g9&y&&Z-9S^@}HmpUdZ^+0m6#pppmz#^cF7YYuzs9Fi+#k_2 zBpU&RL_+kFyjP4wFxrQO7#SXwvWh2BluJXPZw;$X{tX$pAtUNAt0!(UFc%xi?u!@a#_8 ztUmnpzv%kTfGV$TTM~`NhEbyk3dRVcR4Gz6O~nQ%p!BYyQa8O<6I7x!rHV9>E?qi^ zh>D0v@0+S9y-1hp8w<^S=iU1*Kh8b(2z!5Jtu^NybIdW`UAnh!dMKXHWBXA%#eG|b z+of0BmTHM*+l%>xPS+iGy}TNTzC8L`WYUe$Okh#P6-v8&UK(o?Qk#hwtU|(r_ViF& zRsx(wHN~&DJ-uev4o!n>w!hhu;MF55Ba^AMWRCe`;v7xQ%@unCB|H~>L8haR)P%PT zG@ZAX-;)mboO)jBN?2)lWas3^Qb0nH(&ah*wU$V?mMhcz}Pf~Z4pU$U6}nnAWmtFT-}a}u@ZE(n&9@72nV3{QNTz_x{=Ev zO`sfzc#BhH2N2^fQFyIhdZ; zDQ+Hn-$;uH^_Oq9m1lmC$mg*mh;rmY#DQ1OCRbCz7wt6ai8hkfr$e4s2d%cG^z@b= ztw=KTN!rQOZjd0D7s}tfDQvWZrYXkSiqICN>p4UbR|Zhe)+o#sJ% z_ATRy@sOtY>`9|rl6ra4$iw!_4DHxwqSaxp{7rY`#!Jkr7(R~s)AmP)~8S29P*F2;1-QTs7%49UTU01fOnnDDhsN`?P76@rTg z4)kWbTDX}&(EWA6-wnO!jnk$I{OdV24;MeaR&XVj-#0>$7QJ^=Bry4X@2%YY2vLia zW9xl)4s@osR9+S2C~)?^Jv;U!y13|2rb*zrz(`$< zzx+2mX=W}tBQK<|1gj0b@+=}PfVm;sDDkX^c)OvnM?OYu$*uzI{fCU5-{9|AZ4-wr zn(c45hkX0`{yZf$4N)}<3WyUbXC%~aZ)Ln% z z>kKmTD*YL%pfiABr~Fv(Z?S`wz#2&9mXhkWYu8{Tb7LVi`JgHb5}#faqd{p)3hc}E zXOF6<((GBwgD-V~;Opeod#g*y7i7SgAg$@Tcij9;o0(dR!njMtQ_St>MdzEgkuddD zw&>e!ez2wqOQGd*KG%^WwPr;|I-fxy5S!-Z;IK*>ZM@<{!V;Z>FWRZ-{xLO*>5{Gz zMuUuoR}SQ^wSH5#w979Cy_J?yeVXl;2P?&fIp{43@)xvJtextS9BSZhR^?bt5QsMtYcn%%IE1v0H8AVrL%Z+X8HhDv zpl`_DiS*53v0HEH|>Uxrt^;f9;GzoRN+ zR9(V(*03g0B6U_^_vVH98uv7EV&{~6h?}_x61>`K)VpEr z)dB|uDAZ!kJJ+o9$%0dV&QHrrORL)GAlWIQ=a_fGx{cYsBO1dIIxsK-JmX0WzcF0{ zCk>A8Fe;{7RKX}w5NpOi&tQVu1nfuX`=w*LmQ{wtbHph0=vcD~W)jNDqP*dIKvL+U zusl;7hEp$kG(C~<^fOsb3&;v-;Lo~dl#%OTYNvbu>D1ln50h2oxCq~!PBM$@$SejM zbddXfKSgi&zUys!1KUhu5e~JaZ;h?8TaFE06}PPoTI=yNzR@uO60dK}miBJ=48a09xnM4%20qMO5{(q1 zun3lSgEV10hKoUL!=^674YOnwUR{9S(7Wxi-?YTsc&GmEF27W?2cKlyq{_;Lu-f{L zTum+(zph9W!-~JB1ljJWxQH1-RmG?HU@ncvnCgejPS@)EfA}Kh%d_&!T?jo82p!*31zQXV8CGLQKKjeTJ|keFj1?b%=#3bGMklz`$LLdwfole1FZl&596aBeNxdoX`5(jnYn$ z5!V;eFQq+>9=&!iK7O-5>$QFcqaSB0X&+2Jd5E)>C%qobmL#tPl$qkX6Fo@$JB!XVS6xSPn<)!HM7t96aUw|a#B3?JK6YN^z(|b z!3^G0K|#au4|`L)UHMDHgG0IHJaWqP{XXp)0_cruN#sC&fqhJ0P_(-eCUw_JMTtsz}PJzmDyRTFY+6&Fgq2SBG>BD)l#f zhW=^$XlrY$(7x{J2hd^2d%YI#I%~(?-H*V=FnamL%|G_pM}F$pwAsnqo$FQ9nNC2JwE&jxQ_N?Xwmrr=%pUxTA;5Ms&)c^Pj|!3J6op1k35yOvfXH}( zik4b>_IuORZ)WSJx(&<^f>mg_SY#JFC`+4a-Qyp(<@TIYq1J%x3@f7Hed5H%r zlFS0Sl&piW#w7CdjmZ5%d?jE5Ai4BKy4?;jOk(>1{I)pzi*ALO^^sf#6JcTTiTTei z?wMnGT6L-Vu&HB&H_fTi{?34hf9zZNQlL%Bs5esg@<@owp`EF3c^{_o__Jwwv3Q+Ua^S-6dBSji+$!4s(mgr)S3FY9BSr z#UhNO{!RVQ`uCu{)Jfb?Pk2ABDc`b}C{rZm&VFL>z*{7`d#i(&dFFbgIgdz4z7{<- zUx?rVF4g*`1P>@%0EA8)=kPIKZ{S5>Utum;lgz6G!-z@JrUSXBV^1R2mgZf4Qi0Yd zJgwvhfE`~GVmvCs*QLs$Z*k?>`}ly0kC>Vx^Kg>t&t=$UOY~w6928eZBeV{1TYx$_ znP@Mtj%#Zbb=>AZ6M2wPnhAtwehl@+S@M)DSs4@U*6`l<2Lo=;j7DE}bsk`7Id}1z zZK}|?n5o>6eeXo;K4yY92S5=cQn@QL6|#^vG(@4rB$1I9Y~GeowMZ|v*ZOhxiF7XY z=!o@x#$ytB_ms)kYk$gwglQ)*zyK*T?Nw$e~hw?X&{D znNZlwi#1yfzx0$+CGGH^3!UR`><||aHOH^m(19R))KPt3r~Rk{xATXeisxE+ZMwY) zapS^IZU%FXmRq&aZ+@L)~CbcR3-%IRQr4wkH-q^(@)@G4>^NnH6UTz(4n$P(0;sZw#Pp3aI1uXw+Z;8sg{qpnj#4`I zS9E}8u+HOysWf@yG>Q11v0{3{1;mp)G52ZvIQwI^>-p0jDb1&k7K+%(Urw^xC^x!m zdDE7S7H9dD`0>kh{eMal%&I56*FeYwpt;kv(z^|_%V_G`EPiI-m@<4mYw(%2f=S;e zg}GgE1VM_sN>)(yrd8D}3;9B8n!~8LgK}~MzvhT_T?Q+L@EtI>O0uicI}^FC=qtC) zm*I|=H3~Qm9kw4_QBdQL(L2E~ld$?pl-;Y+A|mYgqleBl++-m+xwjoGnq=Vn5M4O$ z|GO4{8zWw4q|Rf)N8({#US|5^)U}-ejtoMsN`_mXXg=L?_j4_K%N)eq%^YI(%O4x-wp9OZ3&# z9zp&DO*j1Z^(HCe9WFpX_08s?7(R~?e4I$IdC1lH)h9(eu==EK_z|>8aol0c&}f7| zBe|;iqLbEECm71gPhVX9N}lg7@!8UmJfRa+vo=~<)^6Mu)=Q8 zzgA)s0Geq8P&2j4l>do+%kCmp>yn#1cOQ|jeBtP`dY84OIm-OKt#`JMO4VN#vbIZ` zy!%d@siFRo%9}bDM8Xz>?y-ynLioT`OP&;o{n=8hE9Hjuy*fi>%(e~H ziOPa0NTYdGat1|4%(*@m6;t!{Wysg&KaiJr4pB6LJM=!@AmDW~cWr#a7kH|Xi*{sv zs`Ssr^#@6Ho0^)Nri-exl`XU(%SuJ2f|R;~(j@xAj@0>Gf&A(XdXLo`qgx-z5qe)uIi9Gk$Syv=diw06k_9%KdC9ttEUyJc@-!dm zz`T%r|Fy_Np%*|sr4DM)nq#ltt{`LgWG}l(e1sg$>rp@Q{?fjCY4oHxJjT1txqud4 z5I8ZgW+zYdm@^^K;sV+WF7h9cSiJgp{!y6R^|g#cjR%BYc5Wq~@Z;;eG&w>ozc`f6 z351p`S_`oWc(3!=-m4J&ejwyGgRr`DtM`%h{g!VikXt^6TdqyH`etGW8@)`QHiO)C z^*v$1@4r4Tw!WG$UU|)BF2UK9l}!NOp?z}oGL97i0!(tmGe{UMb#=v|HnX2|>9YhmEX zpC>$yBFj3RcadLzaYdREu9h-H@siV5ZE7?^%tGjDr-5zkWb2bD$3Hh*ck06B)R6ay zc@ST&X&LbN9A@)4`(-l=njwq~38ZTv!-f6<{r5!)rjuiZH-y48_UzfYeVg?gHUXv- zAIcAS&&yVfI+_}^`eERH8wfAX2VyE52i!(-Tv?A7+=|5p9?1B9ts?!qx2y}netRdn zU(TZ`&Qgmrl%V}Ud{S9TzK^t@27OOFYdzj#kN)bKrVAI=J{)=5)dXL4X zJu|B|+$a^Dj3oYdroD-^kLYYS(kp|hw|Sj(*uEWj`NH3gp)GVxbor(I;Vosmbn*^P zT4O!cG^(n#{N+K7Zaw%Aj5)eRN*A?VRfd@N36C|uGfiW+2o4mD_KbfUo9q9NJKht+lB@1M2hPq(j_`M;&#Fdb57C_YTyXZ88-cl+WLhI=thuLZhfJB(yd zNC=%LT(7MM#42d5iEtA@%Orfl)Y#)VyMe6mc}iZOH~mKCgo71B_g)0t83Bbf^enE@ zLjK298kBa|qzv|Y4n{$4;G|KR(c=1r#X(4gD642C+#O<4QEqzjI9_$9%27OBmEyS4 zO&9V~GB4o&k=9Cd#@hi>~zrg{MC^b+bom8KpR~r;hou z&j#YB5!mNs`h{3yO2!%fkP^^?KcWSDmj+Hyx))pi2bmTT=6RFl&r8dYgH0ItQY+va zm)13j?~!?0(4oO!_%2iLl&z2PkV{lxT2ita@cNCi@={YpY(KeO_k38Fu!}DxJyrGW zji~MPkzb#8%GA@k{nD!u!d@2riEpIK{|lWI!m=ej)434pQ_fZHEvSTb%xbyBymo@oC?C|GETPvdM7g*Tm{`)Z$aF}pDQJ#?jNuI{Qnj{zsUaC7J+|+!yf0K2#)0$f`W;YMNaS2 zSYs-~Mx>fR*6)9F#`Zee)EqqN7cCu|82v5_F)Hw=@7`V8cXc|+iUpCAHSQ-m$up~G z5?#mp8qbbcvQus4!&T^vCVB3zdd~ zg%kH36(C+Q%Gk!d(6WZE$K=Ns^$W(-ebu>h=a|fg+*zJdc7vgVY-|yB-l!abE4TUA zRbM}&rrOS)5*9L_n3xt6S2~pBIk+8o5bW!)F0J3_@7_$&DC{kIJ(a}If`_S4l|sK^ zK$4>=V*hOl)pI<%m$fRN*UdAMAa9XAZU*xv< z(OEMh_(R4xx;li+Pl)4SeeZ`TJ`b61!Osl_oHoS01?s{8p{pIjX-YDK@m_l!J2k^vc2pq{qd8~wEli>^@)X8ihvM*L1^`Yxn)hPDlfAde)xPeJR2 z*8;x3&{Z@1c%TEv{$oe6#B^Pmmt^cT-S(<4j`DCWIVDNOQ6&|QEvw5LTMGXbD!i5p z)d>>|1}0be8F%j6k(8f!1f{>)xs!=F%GWS zS3DPgr89tgD|se#h5{g#Z;LJk%>t(@gcgAPS&K62beZ0N6(A5l-0?MW1d%HU$%nWM zpjdyzb4x^t{#CPQkDneh(a;hJf@pAonxr$yOjOu7G*m+aq}IQMO!k7P;R8DM;!5c$y?$qeU*?K3A@$orsB=(&*y&G~GRU zqqTRQa)f$N4ufM~aDIDr!L2xQoDZu*=|IJDwKbRB%KoR|+xc_-0|6Tbs_#d9GC0_@ z=0%pn_hC`G{uCl*Qtfdjhr%6GCdsfxt{&%}eIYM=V|8L)HXh|TCnF;xzo@L*;udvj zh3A6{$3(cA!{qqG!YsWT8L^KwVJu7N_ckS^e_+yzQf-4@>R{tI$G0uw>XmKB=?uQ- z_Q)g+{@0@9N10>bvTy(KpN4jHYq)+C6YC^yfIuaXj*}<1u_S6RTunP3D~hq)V$v-Z z2?zz$6TB|5QGI@<-}lrd-D0f1j1?50nDHB<3}uZWvD79%7V=Ph!7eySz6eu17yoOD z=R?Z&iZrZ^ja#>FB)gkuZL5) z*l5G0oS~C41r3bV2`fBY$%?&gpTkwmjcPOh-i}UzQKkYZeP#NNlowWabF=Pd`rX0q zukgQ|yiTFgDt4=*6N&nF*$vR0I~{{`9B#2`mxn%Nlf!fAqg8@HxWcl|#S5;@nT?H2 z!+;xoKkdqGWfwg3s0?4ay%r(DfZ5B2f5S>&zsYo|Rr%9{-vO2Cw3H1u#=qxNcwuM0 zhE_qALfOUis+a2%li@%;m!~7A%%Yb!_ba)pDYr!0a$z*$iHPc>HRT&MN1ze-N#s>q zTADS=Pb~#K!mn#*=(aBW^6`H#o-Ed^mLYgI{r4ZyxzB(H8>w?eb42Naj&sb; z)n(mzs2;H$u)<(m-n^2cov^rN-C+t>qhdBF3Xour2F4n zxDkO_p&6g8%H3b9<>!Qjgq}c)h^MXNS(>%3WhkPk!Ok=Ky%id7K$py3tewbW8eYB@ zW>X#%_?)hqXbrCZWZwaRh1G%J85CYyX0#8n4q_`sR!mjH6b7<7oPv{oEkx3F3t4zh z`z^-5eE&VU=sKzm1h!Hty@`m746(IE9V$?ZU2;3``6U}y+kdLkI>)Uxo^*4Mtawkx}7D?U2z)k zQT2lw*K_B$zozciwU8u>{209aKJ}zMrbCd7!}hS0Ebc z=yu~n_7-3!1BeeG^8zp`mbnOTL0A?&g7X?Y^*Zu?BGe)Lh#1^c9DAXfq8?4?%ow$& zUxfQ&I_97i@;a=%U{E#W@qN4$8KKG9L-vYfY`x2|KSFg;Sxs#ijYooOLL+31(D+w> z!jtiP$Iz5}2p2S+b|`SzzZPYz6;!9ZFIYTVLcS-*S@Xa08urTrP*QF`*!sU(A%NIPng-Kf<-o^`)`(I}`t zP^57bBwYgX(=UX_Rtr@oeppa|j>ak}z#KqMJMN;jt?h6vUl-UZ`x zTTy!OV`>Q53~M45|E3iN4eXm6c4$hNf!t?To_!FcQ|L+?=J?^Qlh06AW(EHXoFbyF z1+<^1bG>+SMusBM%7;O&j!pzDcYfcoBj)DKweyRMMdI@KpXgb{6QPv|-&I;<|BDyL z(tUrbIeh4lB0PV{&>#o^_#Z)1dUD2(1xw0gLImH@l&n_dA_ab-3NT`=Q6(*{WI{W` zG)aBMtyE%x8*jZUG61sM_-HUPu{5jg(q5OB=M9R&NvNJ&t)?aiY)_pdfdH$O-{}7Y z|AGAc)>#%}GX#nOr=;Zbrv60eSZn2)TF$#IOb_C^Z(_fzs?w@=Vqd{kklZLlQ9m#K zWk1qYtOXGUJ8E5B9UPhljQpr8t|AnQCQ%J+n7fRpIer%IF_=<(J2_PkadK9g zxr1yVX!-G^j?-YAd^Z<_mZl-aZUPP1+^v=8!95*>+pxswdNUhsd}bYnc>Aor=`DR^ z;650*6_0&>s^0`%DPuRECxGz_Ki=gO#NnY@^4m*$a{57f&IFGaug;cc0B41S@mVpqm{Uqyz(4T!r~FM z8&M~A-cY~c_Jt*8{8p(YMF>h8V6k zY(HY@S~wUV7OtzDlA20w=nutK$oV>-o1d?ancwHlsapLOA279|hrey`W)%3Hg=K7V zxl{t$faFw$SAAAfOVEXw(}QpJn48*;KETWiu>%ehHdn<7+W8#o6~w0tkbaO(X2|U1 zTPJ>ilT$9rbv6{HX>A|-X_%{u!&Dftz+9lYLo`dMIIRQ5UU)6=pozz8;RI3agZ2h` z`ewEe?Z?D);`s67Jl*@m+3`u_hwQuVdGC?MX^Xw(DJ@Uk_6yAccHnn+?l1q&ENeVBj@!%ngm)9c}5=7xr~ums39 zY2hK{Nn%DAazMO<+M013epx0tM1bXonafbLCySNS{X@dSYJ{pxBm-@{aiZb;R)qpo zEfEvM8CiKi|An<`x~?lt;|AtcHJ@I+L%1{$DN_e=boMM4V);UQ_e%4XpPzsX1S@w% zL~$begR`RRfaQw^D|nE6`NB6*2??Fm7k6q{2~$hi7+8kBBeAi+ zd*{x?+udKnA@xLt_?jf~-MMp*p!Xa#tM*z?H`WwYuMptr z3|9XkNy*62cm01MzU5Me8`{>V5)sU+Rv7KW@s@~;3!-ipBQ}Xq*nLekwFL0ujw?vc$G#fbPfpK4T28X} zX?-sl9krXcZyPrzUP$daLnbI!)zp_Cg3iq%u8xgDmfEPC8xk^Nl?(B)!X}N3FMDx3 z;7mNs&wmk=6MBKe;I)zmLY7pE<+v>8x@1~t(%4Zd^;o-FvrWJs;~Yh01+L9iw|+c0c_Rp+~!8v8>(sl$N+Jbr2G zzW13Xyqug3->;B<<3=)GhvCcwSf;j0HuUxN4bAsQwLraN#O?Z+FJBKcUL^4ug)0=! zO`D~!mYy=3kQosA?@YPWN@i2$_U+pPA3SIsTIt)kY10gJliJw=q@|^g!tnu%Oj*1#G8 zj8_tSKAm02Q_%wnowD4q)Qb;PHP`d2`h zwn5)d;(Ag31Y+SZ!pYN7J@~dGg@5Xxp=~!*@@+Z2vfRAw z=_w;4BReXQi#m!hBg3BVFNwR(jY@28ZZ-zdy%3YZVAYF{LnEp_+o&NBMj{y*M_gf> zwE)q?8i!{C1BT03&gKn#MC3?L5EPo_G5eCNDs?HSsb43GRdfjLNn0hqKB}7Y$lAfo zoC3kxG$z);PYVCQxOjg% z!wg2f$Sh>xOtTibE?$hLHo!U@JdBa;*+?tuh_Y@`=oP5$HB9uDW2hi+SNSkR>cYO2 z@M>ni9rKDt?#|*o!l&gw04ENTU+6skf`hvee9wmX#nMn5rD1MUVcmu}jOlG_5aY>m ztsm2_c>f;q=%Jky{(Y{~_jL;5;r%WF?Z{k>abl@L)SD3s#2pmLcyAel-bgcFj|Qlm zjS#*{zx?X9sP=SQE;S4r)MRe?8dd>Rtju!+yD@r3Tk1Lj->#y$LQqmyKQrg9kXc7! zf0TPyt~(MIV(|oWFD~#eXVdoF^C0Nmx0(ECjble{!z;qCwY$@UB!U<{=IVL#LcW^_ zWKzGq@!3P@9z~m?z&-AQ`YRJLd-9jpRouLBqjBc%N+t)-J$O_Rz@9hw0L;Hip3sgg z0|jU%x6eM$N9>1TgLCdO4*uH4BqfR2P2xO3kp{uMdS=1`gi@!bV;e*j`{R#ecnf)A z2N9|f9Cf|6J--0bAW3FOYJ_Vi`TbIbkImCyovFu@4e{r*!VXJad-uDi#$RwDw>Iez zaVBfj*m#JsrpI(8d=^UE!M_H}>s~H_RNLr+X=hUJG7L5}BvLLBDckWnzwLnlS+~xjo^V}XOw`Y-V3r6Q7 zC{6n(aMAmS&&7uaBO|6oqkT0~$S_RrT#~^tlIQGmPkv=>#^7QwzSRJKph8YV2yBns z`^t#f`}lLAA1&SwUiSl+)EKKIm)E7pe%vlGt1A!%1_UZbX?SD$!o5fM_%4tskl}`< zrN2qkl_Op=h5z&+Eew44aHzH`TlZU%)$a3zuAX-D9m?jt2PFiTT|VMaa|FbQyR}@{ z^!K&{rwvgP+fh}MXR@QV;WWWv9S`$(BLpg01qFNc7llbkJ*aTSOdG{eaE{hjH)ubS zffSv6)jPWn!GP);{L0#L&5K5bLf&zrVkBpt9hJNgPAMnM(QrbEJWZjYzm3FC)Lzq2 zQ>Ep?Yq45@M@0$lwU7mGo#chqrD>*jqRrDcvq_h!&>vTite2|N4g z)2E0I1HpLesoWN<5zq2lI|gOf$Hv8pE`2OBgne`3g~Iv1FlIMhNQhoQZq}ZYW0NMh zEUu)e_z2<*vb;Gvs{K9B?>K__BzlHjnof~nZU;q?>2Yv!?%clpBR<->s%9s`GI-}n zP5K0WL1`nhSJ-XIAz_p4SbJX0hmWyY)^T8tlIoZ0jB*64oJH^Krfv+MG~%r0B?77x z(LKvgD|s3)@}uCKl2tbKP^tzA1)fs_2*RVj*8RJ`&;yGL(Q|8?l{|veotRm|N&7RS z&m2q-MYAI`6Q{tFw$8s%h{E1;1=ocNPttXPY@=Ku1IIn~mv2ux#vl z(+74bZf^fu)>_Clo-a+xkuXWGm;2D|t7uua=dLZClY$_X5Y5-NeD{}B z)Fz&>=g3LYl%IxiBa~ay{ZilBK8WRJ^|!Z~oSm}9$WL~`ZQkl?1n+A_zcRvQk<^Gh z`<&xztYLYM$4UcSD~3T~4-^192DZc*U9e!2aWuhPRPPL#o8YFiTo zrPNTRYc0n$xAoMwIL1DH{PKgBgPhts4-b#gT?dWa_x|0GSf|#+#Du@Llv;T4n4qhI$2gp?22ir*nhZB2_~}rE?Zs=t zIhplfR3@!)2XegGlmj9No_u_KcmMfE@$%(pz$BN{)bcPnf{Zns1gS+)zVYMCHA$h5 zzf0LP6xt4tywkD)uaUSXbn+QYJ*;i6;Y{`UX*^S%OA}fP$W3oQZRy|v!wyqvIO*tR+ZCBaxG^iUk=MI1_T9NQ!}^2qyu%q zm$u{xLbH73WcIZ9nKNfF80Mf0PczB+PdjAT^%&e{lJz1c(&gbN1e-IeS7qO)Pa@in zBmv>9U~YoOy_>d_vmX|8gMUU=b$cL(Cl&OgM~?*V=8?qfKy{j|;PM?Kr7HL9y-A!* zUm^g55WDpz1m(S5*|VeV2?f^uG*^*sgc$;d9XjH{cBp>U~=Xfs==sNfw?A7=&)~m3p$2(9OoxP1a>?nP1W{e^eDA)-uMx zi?nss-k>(UAXk#SC9P}M;vyaU7;&vM8bonyFjPaCod1eTCU4h<^kkbx8HgZ(!816N|{psMmFB+1JzwzVY)ihCysny`q;%O z&r$4i9zEJLxU&|vQ)0!M-PyYorlu;ItYtpBP(tz-3O}0OPgd&xOYF_Daz2;|Cv1wi zdF$3-TG5zQ{*x}+u*n`vR*mLzL_U^2R~aH2Aj*fHLB;6JC96q$xS1A*yi7F-|`eV0r4Zvk1g9 zulB8EROt)4r)g>`rgBHARbSog(Cof4wGTd~XDRdFzG0D3Vl2sYES5pS_(BlD z?KBv|uR<1Y@9n2w?wk;}PO~`&w7_X7DFJ8CRg+CywluBTD&Aw0sj6l(N>jl}lgsOI zaX89=wWVgKWX;z%L=`MzNgMT6JmPaZ*DGMh)UIb5h&C-GnGQ?(y!uM?C#I&d#NgH^ z8yFgD6!z#63{rqVa{${w31y_j{25`r@+yBDo0+zwj?qehwPgE(e8Z*=6lwM3%$gO} zn@WHgrK*_$$ia7gd~6-kxORDs3YaKS=UHqH>c7Z(m{xoQjov#8yhHiM7xQ92Jx{IaJj8O+^s*`i%oHo(~#;6P`XPQ%s_a?2uI_iPVtdh|$N ztTSk)um2=QRzVxyd-sK~9#>DJ<3HBYd1jq^B^AY-bu4oYPaPNPLV{HbrG z_$`87iq9Nh_gMvn7J(XZeYF#?Lp*_6ZM><--aMf^zt^!(d(a_#xkErmNY(x8t9_=n zjWx<9Sx(^+-PK*@;W?IRU*FWYuUHq2;&drSjii}l9F$5-REZuWcXVt0RL7F_Nt2o^ zq_lHH`8u2vb*~fbf^smuqScNEnRBgj7qLti&#f~ItTW7Z<~E{e701mz0`^;!+t+`$ z^pePMi&%Ci=HL|C0nSybp7@MO8#QzQylSkcG{03v^@Eh(ZhxJ?2J z8tqXjdiMGU1V}J3(4v%h|2}_o`M1U;ZU0Xh528LfND|W!qX`F_b1N$=^fLPm()U+a zCn%+JqUz~i=T&xPPQZTfQ`1+aeHkxbz9hk@Xt`R~9zh&Q6H;t5-(3k=m@Zf?Mk_v{ z3ql-(xeo@!1u%r-FGgtEj*(%{gY)$_JJbv{CC|X1p^Ys7xoB!Rs>zf9P9VMSs;Y8K zOcosD0bYWO1dVQP4YtB75QfgKd8~b`}V=Aj>Q_Fz^2acMy9`P$*H=c=EcyP=c z9%k29lx3o$$Lu*9FZF7mJum%L$TK#%;8vYS1h&UyhfWkG!XwWQ7WPD2iKMNY>}(BBl-@C6yTUX2KNZsv0$^NbJ1k(O((B@LtYvgN+Uy zSx#S}3O|h{-e4BrjaqFQe#>I1ULD#_PEKBI9oW>m?#s5GPJf@*Q^TM!0X2%2yWJL3 zJ8#`XTck+ixIb}r-^K9yFJ~yJ4FDUa54iM@RH)Kg4Q;x_Y?<%M&CV6`=kssI)Ivo? zMQLy?(20tVQow5xhcMU)&#n8+rV8qp)HdVo=Qw{eM!b{7@5f6o0#Kmlq3Lz>5~k^w zD?@nm7RPKc((bu-d;i@zG&&4X^Kh>itrYN4D5$C$`yY@3iSZ5e$8y3{3IXggOSyIY zTYqI?jI4dApx$NH9OxmnJE?{bn*{~0p4WT`ZU+#2cXaV!e-IuZWvRZ%am);ljsCNy zxMLdGKLUWYFeLbT%%d7gQ3`O%;5c~jH1K9M1l>+POwjZUwTQmCk~zTZKJSIVOyC{N zl5+qr#5TfmZbD~sR8=)Ze%F0-UA6gOEwfd?u)QnGz>md!$r1-eBAN(DKQf2AQB0#3 zL9^bv*?<+$FRF+vs#p=F5T?>aEq}p@sr{o7iu_uVbnl75Ft7V!d$$cBTn<4&<<*z; zekCGd1_&>TE(=;!pH`P_DId7w-J+w?{~?yUfm!3w0VV zo+FKtqFlgXIu@nB(9duw3?jmJa!nGjmsE1wckF1w9wyg{+a(i#a02f9g?;eqU}_~# z(fc%v=A<{FcB2Z!V!%z%dDawofDF2znz^Q-og;(LIPq*b=pc{!gatJSKB4Wo;+c~t z-(KZULPbEZhlRFl0IlrWg*$BmNb8=K{Uknr7yV&1`vJ!JA&Gxe?ZV=NlD%N_wNH6xQHZ_ z^p5Uj-0|^gy2YSt*dnC=W-~cCb3fVPs3SNVvDZK=#pc03J79kTps8ql-M6T6v!M!-@I!xx%dSxK-BgwB-&(Yk+v4TjebK(SFtlm{YKHMFhfvNos&YzX=fV< zjBnT?4O|2hfL=r$4W)8ZoE!t>!{1_x8+%u?$SzfSv=>M~kw(Q2y#(Swq6KyT_UXBj zZ#r08YAAkQ74XAk-cO$Xs9>($JbiyG*95QukCY<1C#4 z>Mhb}phG|~J$kS0BG#CT z7F#xYLLjALSOee+sQ_|;o0zKgf1B`gH$(-`!)$=;os3iiB%oj9=jWfsv6q{_`xYmj z0s2t$yLOaw7vRrlV)iY+O{0>6op&ac4}(N-7uoy#eUZs#hic+A4&)QS9?>5p+Bnoj zC=c13{VnIUsn9TDw*7-^95STBMj6`r@yTGqSJ>W@>IdaOZG66EA6~Kz)^_gT1J1Pk z6KH4_NiULm8pe)AO^hk^75q#j4+&p+P5LbYL^x6->w3=MgFWWY>h>3W&5Bx1Dbte_ zWZ1hm1yAQJ1kJ2k=K$LeqX8o^x*OdVsDbQa3!&Zb8&^PQy%wMBoZw2va|zCyz$}2X zXsg8Zy+XUuq5FJ2d#a>7aR$f}+Y6m2w9=O0qGIFt9xZ?3^M?GIeo=dQjAhTBWa6|F zGpxopIxYS?BV!WLX9eyC4jG(r+`f41BeV3n-kZCw@P`WGd~=v_3qFt8V7`RDh7IP!>m6mm=BOa;KNV|e7dc9bF#kG3aiR3tRSoY_i9 z490xh!UF?ivH6HU2Dk4vkO0s~@VjcYUI;A_iRoHt0CNUjuDG)&#JP<7;NQSLXn9zz zA*#IN-6bWsz+S;nE}57!e$7!BM^iNsf&-y6K-@n=N3Nl4974#JD|s-2E)gz+Cjn-S zM3j&)W^y#05}@avF^$96~yp>2Lx#KG*GD(h&Ltw?U!KAHacmAEwym z?u(#|J!J(ZdcI`7%3L!wS^}X=0#o3u0b&_AnKc9qs%_MZqLYAU58_m$@@*UO<*RiA zw;>rF4#HN*iGoK7T#;s8?@$mSMm(fwMG%Li_GJSoFHB>k+zGeECq4fR1x7l0cBpO= z0X(4JkJKszmJ~1lrtAL+nFb~%(d{9&+9X8;+ChyD#oXFq5SijKQHPpJ9pL2U4MA;o z<;oS(Y3iD$737$-}WCqX5{So#t8iv%YfiM zOdFJ?&ZBQQ@Z~Rd_>Mp-KoO2{C(^wIo4Rn3L~{U{T9Ax@HnJi<4nih^ zct)XB%CnQVFyc+oNqlk?WtnB7zVo}JUrztvVS(&HVifW@2oBgCuJ{CO*DcC@r9h{G zP0f3+=tY7Mo1lsW*CH77ldFs3N?;z~&<7odG>>>%IuHW^^R}P>rb?Okd3%!sPeV%o3-JDVct5t*);>b1dl(@Zl`ntjIdaxxSeHPZIea~g z0D(z5h!$})_EUQSdSs8$Nk~K^CLv;=9)FE4C5M<;`)uJwP*cbim6x~68t1>fl1Wr5 zRKn$#>F8McWKJGeeuowbTqps@!2N~zjY0)1zjyzus)r>~Gb3#c(b^ZSk@6v{2q8uP zqU2r#9Iz=0msrRH(IM_n$B+L-6+3)5sciBneifhCh(-=IofBXnjtf{@kwB$YoLCFd zOdoVklkntPtAkow)qz%|lvUgq`!D(&7;!wA|B{scAEMm93`;t?kbfDY71jXLw6DeEW9!SL!4CD(By0=`tRcu~R&L@XsTppZF>h4juL^cJoQ z?Av$A9Q7nVFnxq}@chG*CklQ618D)Z*C-7;iEt~>*RC`Fa zpmQ^K`J{-RLvIHFHZpQewkpJsE=G@WRnwmqkQXCvl%5GYro>_s) zhB1OahO$kM2IJRB_vbQ~0bk)2Waji6KIeMtAK5Jkx1r^p^-_u`*l8}`Fr~eny!L8q z@Ufa8ZIlom9^NpU`aPe@Hg4jm3Kvx(iZ@Q4ug%-5NeL{K1=Hq@+lW3a zo?h-KB-XdT`?AqI+!?h4t~iL5P(DWc)Hnd>)&nzCL$+DeSb~ZIC@G8YJ6q&`G&Bvt zK{$$x5RbdFWk1Gcp3Y7fVL5$$4O%Dp2T$HlI6aw;8V8bbVLV3S{KfM@8@&$*=gH*c zCh+121-!W)`+9V3E(6qrI(Y=^ZiWMBS)62hysu%Blz;t>%Co%Nn_{n zVl`Y{E+|t)=ZVls_NCIFi=LC3i6B|X@0{7LLxo5X%17u1`m>@x0>@QwRb35MbI>Qm z*FC(tIxQ)w4uB3#+xO#>#W+#bLa@W$V(p z(=L$8mr4rw6VlS8H3b?_=d99jJi3}QV@2cekVDQ-=s2uKxPH=+>|7y};wUGH@ElB~ z@W?<^A-DsvlEvOgQ{0Xs?q z{%?zID>lu|$*BYg925g(6%{r}_8}IL00D(7lL<+J+QQ|GmgB>8^j+@mcC3#aw0H2L z4s&sx17lDbM|S&?I$9jmP1}T0FFP9-&6l`hrE3Er58kaX=K!-RbFHfaND^@B*zb-( zNP&x?8nzF*ScG5sgYF1@G=3%@?l*-9Jyg=4rD6|}&Z!CNmwH6H2$ihg; zWa68**h;auBHU6H9rx!r#Lr8S};Jyj|GoFz>dI7r82Y4qa=8=(;`C`iQ(Ar^609}+`>hb?oZWfu#JnOyJ~TAjfbqSx>|(50>4 z#QW@$nzb^eK)qqB+m|&7G7NfB!Oi*l9q>6DMqWv{nOe`H5Z6u6F^1mIHzR%H3I;uw zX!|b5@-%MrX^`_hH6hLqi{mBfkh+AyC5739NF9*0WtzPo#|!RiGpxHa5I& zcVw;jeV&MCzt+{?IzIU6)&;e9$_@|POjY4~g~vHU1FOj8cgU_$TPAG~-7Tu2P-SuU zi7aLGPcsPP%NtC96kJKobbN=*;iy6N-UPoQoT%x=vkytfa^J~5F?8{w;Ru|h8c^Y% zjfF-mdz+v_Su^1Fd|`+agf3h$i{qzlMWNx~$e>6G20yg>3cm)$@i}2fT#h%81Kbqp zGAWZFb8_(0DGyTg#bt_@Meu<jZ``kPBhjK-5?a2J}t-S~+>^oul zp6*+AT4TZ*IlrkDY)C2Oc=#8S(gP;SHc?SyAAum9k<)FzSQjR}s>yyD{^p^P@w@t` zm9dYTKGimz<4e5n^+(>xTE$;m)lXTpb?@7}$Dli_XubNSs&Rw(PzRC4$w>Q}i{kyF zHO@VB@Cx0MVB#z~Aalss(eW~h(O2SO?iHO&ZH2a+ppuEZRq^UR`!%bL>(2bwuYa(z zUVU&K^7D=MDT*LU7yewq_S>2@Yj!RlPi`5$^-fJ~gUq8M<8)rckqn&L52~(@v}K){ zECyz>e?9_)e4A%j@9V)m%SV%a+upr|Dj7ZLfDmx@+&LSKGNF$CkG_^A-&UNX*>g2h zPs_?~w4pE$4G&MCz@_>BqobqK`rK6v{~Nd&j}6_9+2;(RgU7~Ktm_AY$J%x4XjcHB zIrRDn-#q!a&hRyWaWrNL`=`~(ZO(Laf!nfEbYX+8nHkTygFC=!@U`rH6aD1LyJm*m za%@sdP-mnQ8V*y^L4xAa%H2$tfwNC0hjfQ(E1R0~$HIF-46JQ#zKwJDz~RFia5MfV zwRCit!9d|KRlY9BggQ`PYq+c529*RlRF|O!D(^mf4Cj9jFqR!3aekfF*4~Rgq$l$3 zjV%Ee$FzG9r|`Jmhl}n&vL)g?%Z2RuADo+QzTl0&Kwa+TIMp{}^4vFnk!FpKeRXpJ z>F!QCGMinvaN$HsVIdP}P0|(?Mf2wxgKZBNT)Z58~l!RJYSacC9?4P6&-02!`>2iMhzPkDbaEY$Y$04OJ;DHxlO z(A#DZmBSX~T-{3wcFNX(ikraZ&Te-`e)kG5i>+Vv;>E`z^Sgfk{i;?s*clM8s;Nn@ z8BY@HZ^avep7a&4ll+4>Nk}wEp`pA#^qI}zMRaT5&dzY>b^MJwQnq>+MelLIr4202 zbieEDv9cTQ;%22GU|Rs6X?+gjt`IpG(kD8L-^{icU`Zk{v-aN$3-dx@8+}UN+*OCy zUeDQg+RXlbVBn`{}n{4tyhefXg%X|Cwt)d_==zI#L!3@}PYL6msYMV3poUSwZydeJ0 z$G|*4IlaEV{^k}xT}x<)Lu6P?##>cw}3K4M~z}?;8tWMfP&M=oGPOx zfXZyB_=H&nz&NMXt+U#^fyC&DG`p=S06uU9r;|U}rhnCXHAc!OynL#=m(9>J&bxQ)nYM$~35}S#KR7uz0hZ?R zE4KLXv)jtN0l7%K@f)R$0S~%L+@)Zu%-V9|?Ac$D^4%U88F5pg?AW$#3%lwe{2!ga zJ_+#*4GbtwYZi3HR|}mOs_TAo%KpB|f#^EJ-*)UcFz7++>heWhA(nb|&zLo1B9iF+ z!4DoBa!x6|^a6q8R!gOXRacm+lyOYmKY;Yw_Anr;I|{=pF|x5+ z4MKaZb~d59d1*PTQY2jwVJ*nyLoXG8==)N$yu7?pw*hS@!{htyM_Y$$`g`HCn4!+) zuMU*oG`$0YUB3$d5opoRB)0I3m6i2Z>*S=Qc>mII^bYBsF7#1cmxy=6OpH{av zT*Kz(yr<23F0LuGs;cVln<+tGO{eL+Tjifqa&lDu5!KvRkl!&JF@aqQ%h}Hv8+FZ| zK@tsF=V7LjxVW;xxmPb=-hALXGiG7kzc-?DGvm;l?%@IZ8l-EE17#?zL{w9NWTX!$RoJwaiw^jlNQKhV4 z!IrO|-eei%Xx{qtx4$Z*N7ZwKeoHpr+Go9U)$PT(J4M^yS+!hI zQ)4JBEWFzCoV{(xc(b!+>MnF&UV$Y`XY)<|5YAxfq~p}_K<8A4si5efscF_K)CYu?i>>aG;nv0Upsd4q+o}5vY^L5 z|NL{?d-=8&T?TTn1U~~}qkh5^N$~$E@56~DO?Ckm}oYs#mIvg&Z3@GZ*tsu_`_2*+t<0O3RjM^Jfd zMtpz1(Qx`o(?Q=(1x`hV&9P(aPT?0jUq;M7@aFHw9}7vc-sdn5X&M3n?H) z3=Aef%u8hLJIc0Q?@(zYPs?kODDI71f8zLYXCS0RW&8B-?r+aCw`x0je#9le7e4dZ zwuK$&Ub4=&5Za)VvR%Kubie)i* zW)PKI>5Xh}C9oWfrF&hlIhGFIY#AX*xc{2jFwN2n42G!AG66dj9ON;8Hrd$#UTbR~ zaE43@`_W+3I5&t&UDivV+J{M1DcPo6KwSs?r^!w0mR7XH{sfXI&!aG(9L_p`tjQ^U zw}5~)??EQiqFa(e1cS-=h~vj~;fawtT_bRng3h|0tel)g+A~18(7tFff2gV!y6(TM zBY~ngoeMmWjG%Zt|u)iw4TrbnmTq237qt>OaAcJFN=bsYkgNN$rj zMgt?`D*g7-3T88t)!WxY+2MbSiHWJBmEAOYB@^wVsvon*%ovr~*@AFc#Wfu< z!Sq55fi(~*zHJa8Csy&A5dl1ucASBlWOr_p6YiXn*AFDo>2xQ(EI{OF++OkISS5$! zexYI-%O+&3qyMwKii(PxhsT4J7LAoBr#`h=JDPR+VN9yHVp}hoVq(HSn(^OCCnh#N z-W4=K2OP(4rJTk_O1479Hc&H$lMU(_+XHgOW~4-gp-_yizlKddjLnwKAy2Jq`HDdPDoAt*ixyZAcX>VZGqz>2Zj}BmU^uFL|kjM5StRG z8i#^roj}Lg%r+5EK+C*9Do;OXun#-n5*NF0_4hjH7>qQs3tC24VPCue&P^HBMYnF9 zCuxQwQa<*%-=o2epwn4sd((e>8)AILV)lREx>lfP!#-#TlTi5TU*#bxu)sQ3qVvTF zg{-mghGl5zcK8QUc#e_wfzPe{N-~(t6m+<9%F1N1gG%agxThj?UFW&o6%J8zMI3PB zuYF0DS$aOziXK8eX5En4$q+0eX+P!4&dA=8-$Wtux2Xuck#h}jC>jd*i|OeGhI=q! z%<%E0r|O?U@lu;?5)Pl#cgh-K?W|tCVCVd&)1)e-9gT};6NXEw|HjefDC`Po_ZQ=4 zky=!TS2d&|F^+~EGExHerPP0(nAq*&GQhId6z(}&1SO6mRnZGKCJLgB z%LS$Bc1I?CE5zi6aMhG!)|DL8!Cs+@N@4K`BfCXE4_yQ?p-YGU!oY5Ogr*`EAHSZhZV@(akTME;0&AhuqW1FR&xef zS_I|JpP89w;f2Yo2eZL0A7X~HU^9v54PIRP2MbIumMC>PZ1}~qXOdv!xE32ZXMlNr zFk`ylag^jyAS8eMqZ#3cY2u#NwJ!#s;I0kCs`(o_M1<6>->Ct2F5fZ*yB|837a4gXcq-FB?Ji@pYk0inI2AwSQREVR90n6G_nwW5*8v+0RFHbaiXowq$cSN?K_P@dm^md4rIG7R5tKN=jdm z4d&>|U|)3+Ge(0~rkTg0T#R>}hl!VLu$JQhMbN9EZ_=paRudaFVR&c#VRB}I_I{8( zHjsDl2b0Zj)YAMovDMYpyT4`Lm3jJ84m2;|uAl%ESAFoJDh#dwz&fAjvII2`UX$Vo zG*)-okQ0}63I>OS41k1{VT$ZEH&-Zgn@lswZR+TpVUaIkC0=pHw-GPHT3Dr1zR%65 zBV(hw5|@yOPEB26OQXT+q*W&NX=>^*B$4q)@BQ3>N0N}7tTcWU<8E?~B1$ddYuBJ< z1Ufq~%o0QlIs&p^#K4oOsm=WFJ5U=5p~q%*-~hVcYT=ehWT4QC5o1L}L>5gf!UQBF zWKH7NG(}R(%p_o|CE*~OLs)N1TFxcpZ^ULnORuT9S?IzO5kEgaVj-%j388l(g=iD! zt&K`xJ1{NDW0U-K-e|n{V(dE;=9ynZBu1ll&~4cF1InsjuO|28fBg5K@BRP!i~sXp W-PMu{TaF|V&0I56ipgEFQ_Nq}56^c1 literal 169957 zcmeFZcTm;WyFN-{l*C3ff(=0DDuow+lYnUfrh-(A*ct@nM}yB^A3mE5*@-)1^Gx@}Uw z|0YjINB<)o-4C~a+=M?FdA2tV{}Qp2P_>n87r_I6E6PlKSWpFIMODQzwf14IZb|dX=$(C-rkvB zZ@%F`4U1Y@lTP~IfB(L)uyFMFaan4Tl7g983d^nT@2bPjsGL51ns;w*LfzT5I}th_ zE7bnUSGMvB3iwFIG|B-X6aU<|Tok`Fjp*70>40O22Gt?M&6&|GE~6)SG;%KEHOpp0 zBO=mQmgY5x5VQRyw-d z-*aHo`|mAuoDkl&m;c@8zCK+)y}4_T_X%B2FP;slc2A}36?46nc$1sgXTL?>i+8gV zMblR^-9Ch!$yK!0G9PWu+#dG!P6XbfKyNvaM?*x=B22{oa#awYV@2~u>FMPy?(Pfy zyS-P1&6=-M#`~*+wD#o|`rn_wc;Nyz&RM6dw)c34yy~XelBIFmw!CrW){%THP~3qt z4$Y&EukUQ!KG#y{XjwnmS3c&ol$zVgzj4#1z-Gfz_m#^JA3oF`sb@(mdC)-%5vk8j zE&YSzX@#KMqFv$ADVN!lw*!5lmL(^!M#=9=l2y_zLd4t(-md6Wt9RwBy8RUppg-J} zn@$zRdkMzI$A^9l7d1_ieQ~F3@`UdDX2xso?j?!Z#m-D)ZMl{cGqx**N4HEEqa3xq z--_ySo4@zzHL=<&Cp%*DZ3m9=@zuP@k&=>H{1R%}zAJdeVYKH z@D0lvU1n+}X>YdhaS8}zzY`1MR*U)=c4p+=!(Uh?rq%m=ni@i}`-I$=Z*@3!xrsPU z={=-fix5kFa#%9ZIWQ_ZD=SOT=+i+St%8PL)q#&;6ukCEdirD>K=Sn^4!9Urt}Rd3 zh&azANG!O`b*L1)LbbjA3n`Je`MU#Dv}^l(#mBSQ-xdwkUlXS|nN|3)OaK1+*Xx2! zfkj1UMOS9(JGXCW%D`b2o%;K5=l1B!A|9(n+D`q9L9@Ys{S}GrT=``(Q6XMrIrh)H z^b98N{@lSIe3@lrTD>2iUxWNjH+;*^?QfH0f_RO)U;Uw;dGmf*W|~%^$xvPF=FOXr zaTm^Yy6_Cg#>JVmWa+c}=#N!xTU}i(7)dK+euR=k$G%?0yY;gTYZ{Z&OH630s;bYl zT_+{*@vezrJMik1q}UF89P$-k{q!MNfcEt*1J~KJ?QeJNw(Kl&60!aAh^iamI`z;q zdO_yat-Q+|3J=9myHaG{Cu_=W=!+A#N%At%l;Aj}jbw-}w;I=pvzc^hZ9uORyI5T5s-dUtpBRco| zm`d7c!?W4S4<8iQR#zw{)MMVd9fgife(W+I{Oud!8iA}F!f_IIe&g4N!FBICUEc%dX`W2Ox$J88i z4e!|qx7lFHMeY&lrI3QEj2mxXI!N%NnPu=qRDmsHu^&=`v>_ zI@_%4_3)uwnHM7?GjmFFvug3ucp=W)rv$lZz2keH?<+jDckkZert}E(AzkYf!MGF+b{ef>MSpBT5}UQDbiQ2P&f^ph}0=l zx47Swl>@hKx?3IL#mIH+*IzHBYUDN+&vhoZWE+mu#mcbH7NW?A&V0IP(osNp^Y(53 z#6&t45)Jd3dF#vC1CH%4nzUW6h_B8^Yq?}=nWMY!J8|>gFg5uIni|;`qAs%yC%asA zCwtt~Y-x2-j9khMFOBQ<1PfM6UFW(yO3O_5{qjpRPKtLyxi6~<&bq|PPrq>FolQ#? zGH-1#$)_}Bp$smfvMI%1JrKGA(9x_}Lc=~~(zpHcA(PXzDJ?8S$kagD;P{@DkMm1F z5B8t`-jO@!#AE$gs91R!kc3Gmd~$la9@~w_;0q%o6H}aVeO$gd;~(C;`I7MR>bVwS zWtnIaL!PYevUp_XIk`C>jIcZZ$5NwDFF%l1Q1aD4)j5^OGiBCB?rU zKCF~^Q$i_8QL0JHG0d4Z9E&z;f}^I~=TPnvl5HLEu4V4!^0F(B`pe%LL}&iO!+P=O zpFa|GbM~wz{`3>da$0rh$urEmfweAP#v}e3V$o3$%%--SqM2V8+o!r;cm8V+@5l&_ z_enI)kN_Udyo`+MGVk424jnpVP+qz+tC8#mc+|vRmO*`H?YRDDJfFmuD!Wqkh&0tlP|U)y}SM+O{t*#Y3fJ9G&+iKt{5gw%a&P zf*wWNTAX6qB%G>U+&Y|>n3x!cVlL9q_3VT$v)7u=+G0?Jkzng+y2sioZ~i5(PI~N0 zhl0}Y=h|(3)rweMzy497#~+NkW;m%iUN|8Z{~nb{P{^{gJ*tbI)%8<~wzhV7BDw_= zZE&kJ{{`S&xnLGGRXxlX8;v7T38BM`hjlq4D&A>Q zTDrDcNOdn=8b6aetG16oh=q`XMaxJY>*37KVwX^|1S(zR$x1AZcNA)*h6V;2 zZ2s}bjD)q%Pp^rrFy~r!Srqg<*gxvNmbbW&zbIO*WSu`PYL3^9OGw}>>@8TCZM!m1 z6T!E#1yH)^!CpdbwEY4CVk10OTcV1vV+#5MHHvc@l>mGp3` zI~t!U#TAv(Vr*Ky=VMbto3MGSo}2?75kcbkhz;SQF8)uRyui*F`uoY@5K)&LY@&Gl zX>0pLlN!!#Gf;v`=U6VaDjU773R;^L}`YH#V6 zz(S?xn75gf`y9AR>r3eEtZzRMls#cwAAe0H*P_F2@YA8f_o3n*CFu8t_8dYuhi&Ub z^OlsnydRE=nVdV#CalTJy6=6k=qfgpefO*Hq`p*KkaGU|njUp=BC~n)33?7Ti7DVd z??mDCz5L)HpFYU~3)bE|O4%o5VpJO?QPAgeMpMjPPQdUZw?m@f-PhOcCpabdY~3Uw zDQViCXH9B30i9R_&NF8;KzD!zV@PAd_YI{~ef-!c!&j`Q9EgpW$)-gcrc$hWF2u&f za2`1#8GkiQUQ_cqi^uY$JWno=*2GYwh9wn8)*l-VdxVppzb>k?oYliD=dy)bcSWp( zcfokU$V`8bj=I12eD_`H%ajSAN?2vl zL?V|75qsbH+xQvfD|;%f`O+gZg@Mn}y+RHVCHryp0dn`teCf2qtfv;HD# z-j-wT-2DeVx1YD$;!G_toqt$ZDw@BV>l1U-<|fxH7P{Lu6;%RzfRQUxK@C;4l7;{l zGStNn9eVLjKOC#Vn{1v+weQ~<4J;}g3?PZ^Q(cKRWl8#lTClIL@6cdvwD6Km)6VGG z)q4*fgviUwYwo=Ghu?d%wwzSst`axi1NGt72a0`0wQe?Oi?7T)mC*U)k3Ztt+uPrW z;A>?iOxMZqwTud2r_Nwog=)Jm=oj{t;@vN|w6qx5_x1LsG>TevpGyok{Z56i#>mvT z_WNOi;UQd}kyCWD)}A{h}J@!=7;o;Iha!zHxGwOT)Smr=HTRmQL zST;o^+wc*3z5o{s*2uYL);O z0|P@-mVVW)efvxtb#hk6+e8d%mi{=iSj2Q{IwX+2F_7xdhObh*wz@p$S*P&qD9|n6p1ou|JKDk|(CF&w`aI-`!{Ur_syd}I zieRAJ(}!z-4_V#kUsS94rQNz_E2*i%HRhs@3fGvXWhyrS2pf=|p01{L?%Zt_w;4I_ zf<}YD4Rp7&yopqIJ4i8Ex!Pm_Jyn$X;Yd4{-k?e~Ox!~o1m{GufWxS&@4KB)h5)UO ziiv6SmCQ$x0}PVNB`By!uw{INd(y@&OseMlckdo}vd0_Uz6l#iwB%&;na}itC<&BR8+LHO{eHq=GQs&0Ip-lDo5vQ#Me@w5mC(Bln+b#G;pOC zMd5>SiHVgcPIkIZS2OA3kt&b0E*|fy$68g3J*PF)MX5~B%{2i!nuqgAnKUFkDOnum zVz{J#6ulJnxPd3Pz4w!8o_-Y*#j;DAo`FS>%lFSeKLK4f#7O;;JZ2WgV5&p+R~p+C zq!}+#pT7}Wu6y%lL$={H z-*;0r9;*t`mmVJzzEPKJsY4J>lq)JB@8dE(en7ONe0qH?Hk8=xvW$GUu{EfZ@r2uEo45Cnq>6I=XnL%-aXsXl1)~R7C;e zXHe?ZiW{~1hfNok7p7Z_oU#E=_2e9@0(nBP^|e-bwTn~%I*#P9s&7XsY8 zr?7)V-EJF?-L1>I{^8jPVevnqBANwjbbf60v>6&2VyX(uzf)_Z2u-%3z}~cYrcOq^ z#5JGljHltS&>U`qGo38+GRLArJy|*R*k$!+mpNAOJRE3FBdNJ2KvqOc*|lens#%9_ zxz7+fyD}h7%E$1mxSM5OO#1p`BE_xLrU8%`bJ_+?(~2+TF?`KS4L;<~c!SxcGGK5&qxvD~h-4<9aPPXV?K2I;sTJ+ZefLGEyJ zb{v#q6G}iYRw-Ow7z?w0#+KmSM$&2jOj6F7KQjtbO}} ztq080bR7b&fb1z7GAO5Ms^xY#gkC&R=zRu=E%oJuaT$-w=%4=nyMkt5EVTQs*QMPp zA=|$CdMsZI;?tS65=`5H8ua=uy@o&i#*G0cmtIjH%4vwlhB4aCf4$+I_19lN`*Ks8 zOWa(N1(L5mKXqfaxntl}0I7iXe2I_@5?oiP)7MxP?-ejGn)+NJ>|08nSE+k3{o#T)pk2eJRCv#bkx(*(M8+B0o2Zr3=M6U8yR7ZjjvH zXkZJ0sdjkONJA#@Of(+r`j%jqbp%iZ0%%@2S>!S|P9zkGg-3e@1YKs$yr<4g6xEyf zRqYTmX~<9?A2gFuDUHuR#_O2D7#eM3YpYeqKu2dKzHYT>&p<`fSjlk05i5X9!DEoF z$(1rDa0;T6@0gpA{jg&3e6P1~P$yT0-H^O_dtPl+d*dUYHjBdQn-3cEtoyi!iDsyt zXLaLY*ZVFUSrqQ;a_&46I~=}z*z27oVn3`}{88qqAVf|Dv}${x#t~5UCNx+(JecJ4 zwwpX=nFt>i1rv=~NIl$V#kI5l3f zWC3DZmhu4=^=tQQayX!6ljZI3SX)^{DdBDvhgilkG_5Wh(ln#K1M*DactKUva(0ba zJw$@Yckh0~DqTexs-K_i;~6e{_b$)Ar5Vy^i-DgN+LptSpHg`QOw42N;aMju#9xiw zlJh5ID2evK8jgZ8bSjrxX$y958B=@<3%VxZpLR1ORcO@?X{*gV^usq4Hv zSxzfmTdC3N0XJ|wj(@xw3HMladde2>vCaxs&eRVG)SOkgsRLrbOP$VXB$z^msv-xy z<9uSoI^Yft4pv?|AZYZL#7d@axq1T;)Dp|nA(9Tw4Aoza*zmMsap3^5WtQ49Zrr7c z0X1u?9uX7kGznW>Sqk0GeCm2-WhFma`bS7+3ZVntz^n#+n>KC?W(&Tlt9xalbgg6v zY+EQy)TPmGVM<>CKH5=U-j9mu+M?d>et_}P}k zlF?e27~TU`P+;?8CZH9l&U9qzN-9`bq#Z>ERRo*dsq>3ue(%I*Kyi!O2^@>(!_yIK z%kk7(@t1?;lUB*{0maR!M)+9Kghtbv8kD;}E^iYIHd{N8IX6I5IyVjuju3Q68EO}f zP?)pWYBmK-y0hGwsNv}yC^jPnrX?3`UKL*X>S3%x#_S8LQz#aEgFA)}qA zuL9s(Sxm~W#`tYW+# z_-ASZF|Lj4-nG35ItpRVd39lchFxNcO#>%^KzY(xN$a>2EeC~K&d%;71g`kPi4u*I zowuT^-upg$So{36?ab=Zgeiz4ku~6Z$D*vk2)%ax{COfD;|n*$UD;2Y=+ZgaVfVD^ z{ag;5b#k%@8aMdx>;)JNCl{9tfQBq25+N`_@Tbl%?lLOgzP?R>`xHKHq>-pILFFcn7BV-X^2+=RFHV#{${U7j})QR*uh^j0!)F-eB5+m7?rK8|0?=l zBdf>qivTVa6R3!-1@_5Mwc^lvVLEbxUxRUOW>`;Wppxzfi@bzgLQ{Q&y5_X$X@%aD z!+MHFmPHXNv$3(UU)l$0Cuo<&%saaGq}>2`>7Mw5{Sx-!x1axK0y=-bhWcN00Lu5j zKQ-4k9s2kG|9>d=Gf8t;f5lZXS2vzD?-Q=`R&MYnh1)bsT-C&vzuslyQI7$bx&mKc z`r0)WRAVdxJK^a#DhjA*c72aeCSCO0o}`=_BBN^+SfN)3U3~@{XehVKozy}Kii%dA z)$6G7q56167M5|@HMGY%I7$?=7Jl?`RnP_{E2|6`LeyB9-2}g38*!g_kC$b9pNEHz zUoXdBi%#c0q8;3%_&;e;u<+-)@obRj)Tb;>Waf~ zzvy=D`Kj7nyLKtwEM1ou+1Xo04Y8-UM+{(h2VV_8tCnQTZ}a*0(3MWqzu4hyHooE+ z`JO!}5w-=TpqK9(zzz#*R3JcHChfCh#dam=(FiL(ixorQo{ji-HzSsy4hl>Zk_95| z!U$q)+Yu06p=TpgO5ArW(rJA?J%3e4Z2#f9SZ|{8({35x)XdU*DB?Jt5);Gb*;-as zW(*%`yYD~>mXUE)%;;03{M=Jm;_&lFpk*t#x)ybe_4f4<-a>- zpZgXfd({WRA>=$WxbrDze+xiX>NT;2cQ1rJRu|$gJ=)V#cWV0qkr=dy*70?%5q6m7 zzPq9%Cxw&BB*trg=QuizV~2eZ5=ctJs`zItK&=dHKV{$l`SN_Z6zg$qq6#xPo718|j0M;B^l3J3c> zVY37j3OR5Iky~FQ$BXB*y+z=LQL$CMA5&1d7Ll`hX;{O8sxt)jjTjEqwr3$O;yqg# zmym`r7WS^Hsi{eG_;^gjB?AkYq*Lmixab6>gmL$7Lph7%djuh$y&0JXz#|?0(&K&X za5Ch?iFZ^q`G9&NZ`He;27*a<>$aoHDe{27I{tcQPTXaT(!@PXED`^R2pzLEQd@^U zo|)jbb;(f6MmLb3cr9#P$L8&@>F(>7I5pYMSh6?(kcub^G)A69^1O!t9t7Fh*qWJ7 z3JBB_1xMqt%feJY`}>u*4C1QqYSB)7M_)G^%^bIw0!Ykdi{OAZMLyG6hDCDW@c?du^FG z;f(vIsWM!fe)z$t^`$ZYzK*r7kJAG+BiK#HTwUSE1|&sX6JKc<&Wd0Vo4=kZqvEsY z#09{M9L}1*|Ndkc;btKBk*0Gt%>y6t)yjO(G4fe`lT|aLB#;!S@(&FBa^I*j5S}wI zRdrMm{;53DcB~6>QsVn`bad*JI>m&sZG1SEvFldfHZ5$^Sg+S|fYM69{%nbO42Tse zB`&Jvm<3a(1|mF4-sR*BbL*6}M?D(p68`qi0^g#TUr(93xE(lqOBVK{GsHn6c(ZeG zI3MDlIAb^X7qmJfISbP-3g~PpffI-xDE4W{r2MdPBV!?bClvEYJdWLB7Vy%;ore?U zAM5Osm}6xYG?GnR{2O1F;Lu7UVhtLEP-+S$9`(xkro zk6*w2=FJ=K${I9mEZ;epOe%1-~L-dn{t)_?)u=`h95m1gg4`ldUbrvQj1AAj> z(<;@#&%o!4Q&oICl@oBQp?|5G_l?Ce+S=HBFc=CDKL5<$KL%yjdn!4I!c7<{Pvz9q zlpU~0Hh-E&+qWW8)2)+2p2^uX`B-Uk+@zO(UcvcWgyt{wMeE?xJ`8O-7f~&9cPPQ_5#0csZoQ&NCl3N zr#9;d~!oR7d$bzJ4=D!n$qg4P? zG9V;m_>Uh&YZ~Pr;7^pRu5dhsp3Yqf3UssS7Gw?;9CR9XT}S}ot>iOP^#x+HvR~e; z)l)NuBxA;R!)d(zbm3BlQZ9C?a#j8+&p#jFb*Uz)XaQlKhiO|iD%dn+uE)j2sfY3CbM+`Spz@L??kze65g{H%y&A#G z)j4?H(~EUp#A>Q|dW2ny@tC44YzT<4o6^UT}Rf&;Y9?&AbktBS*?Wr9~<;G z|633|^pj$eX9W#cgEvYAuGLAlcN}jY>KQzH_}DRNAaACANgw7+^)>8A$0o~~|gJnn%o`D%f+OL@+A5>8Tn^lELt z1iVHQ+u2~zJ9qEmWOI2>iLWgy{PfdL>^b7D^BF{Jkn1FMEngcsu@ECR^5E z@8e6NtMh%tFe%G~7{&M7XLm6%F%k-rSOC*aX%>VQe=H5coZwfo=D z0nfh;5?!6-x?Td&>>SZ9u#*DbqHw~Vv63{>e}fXHFP8(SZ>*!RXsfEZv=DLpnv>9}z{uZHRM8 z#7AgfT!<8q%*=bA1IaHgdL9!O?~3YBxF#A4Jwesqc4fEkw`CW}E9L zC^I0XkQ2HO)LLSsp3Xox40L{u5Yqt>yWe3{WO3o|9E{HVJRvDI{XxJP%#GQ{5I!=& zX==e0dRZZYQwr92L3wDf0Xx&SHn(e9vNTYSFiY#g*}nlY9bS)0V7Q6edI3 z+v#5Pw~IvJ{Oi!g_FZZ%H{gTe`3}Q8%P!#Sq@tR{Mz}9ZLG+@Xv+E>Hw*Si)X+Kh* zu&WAQu7QG@B51uhnq9-JUhonFa{cx9D84NQqG@Y>|Dc;Pl}7URw=_wag^a=p zeU}7a!6Ar&|1X(=xeUt@G#NtR0T^r-NO%$&B;&4KpU1w6zQ2Bauxo8Kmpm3ix6}*l z5=kZl6lCMZjYs+U&*@_ozsAO+YriH$Nm|l7;TR%V09uTSh!5cpx$V zh`fGq#FOCccEH3P?kp~Z^+!mLfAh(vQn!CF`VEG4c*d)6JUN*&y4I*^)o}dLYweHk zhCk$mt6F?|qJJ>;=vaP5O-*jI_s8Jr=Lmmbw<0JYWWK4%{doPzM`S)$p_b44(JSLjbSu4(yKjGYH@NFg~ zZLdU-+05kRlh@b3+WbcTlhxS&Ig`SDf(!wD`}u!)LZzg0J6#j+-Phe?|1(XqDvW_0 z$@e$@4@Q3c?$1}J6@~ZD7fM~Z62JzO`lP65&gppdbpK`(DXv**D(*wiN=%EDX0mnr`fmMKRDvv5o zKrAV!ga$yPpw5y&10r7BQu|r*&)#L3+wq*@w&dUj-D`VpvRw27B;-V;>i*nunA>M*X^9IN zXs9eClZ`0)*svp77)c(pOA+`-l0TlhT%^8-AS!stmn9V`T{Y&?MV72t&6FW1h-Hm> z`U_{fTYH9{%itlX_x*my3L9e>*$y^i;I8|nZKkA?3Yi8%*oj8|z}syzA0jT`V?!t_ z#3dHI@#cr19LG&RU3hhR`6A1I2kvzHMRMTi2CP|Mt8LlM+kpLdz+yKeW9r;!|tj{b{;go6GSOLFT}k)=35P+qA$^+$O+?ka5Sx_@eaOM5AQBpxuJIAw!G*`^ z9~2aS{_dvJ?TxLr)^VvPYc|L}&ziQsZ~6&#Ha%51V>b*83CANTL=LbzX&oZ2|es+HF-sjJy ztKFq|S|Sdk&xl+H^qB-3f$*}D`Q9z;?CgmUnaKPgaRT261MYGzi?Xr3`*KRcm(je> zY{{>=@QclPzg@R=ne$n{Q|<+~aKLW!UshI2=MadnB(0Gs6##YQMb4 z_;UnjA-cl|G4b;8$!YOTb@{u|Hd9hfgyEc2yrcvz3k@&B305r|a@q?MjuLeh`*TS%lCMqJZs_f01Q4omPrC^ubr)@6Tas5iIE)1mmrDxho z*f^dhHur55*!uR(o6+%KZnLA;|5tjf{wb*CZi>z;M_g0iDGnIi3@`pmF3I7Vd-jDL z0e7X-RPX$qQR(Gk?O(sLOX=(0ENE;?i%E)!erg?1q{t8q301KXjAc*`>L2bltAngMe_^Q}SF5ecM zjx&9y0^i^*=@6Co9}Aw3P%=Ai^uBvYcuovgmDzY_u?85jUkt~U5FgL`K@=6p2tR#xOVdA1HLeiP#l3+3d=T((IZn(~{^WN12g?JwjZ*0J_Hy=ub zcI?=3ztqscfN8Z874}UR1M{g!M6}h-J-&9Imr^m{ZUl|f|F4=_1yW){5H%rAB1xE< z2^IB~`;Oe-tu2ePM*_n^?=x`_afF6)+AI`M(PXIQm~@B-7i~U^p*Y*n^2$mly3zA@ zKK+ELDQGybOp`a4ZoFO%S$vBJU%`7Tm&Bmek^&2SfwYqyCpu$IH2z%+l1_Vx zDKrpu6(aRO$m1Dyh1-rU-(84FiVr>=P6J`Ye<$39#Je%&C3vO;S}?JK{RH*?{@J_a ze@bt^oSv$FJSIgCT~rIQMMso<-K6jGbKu5V4l1$8Fkp9#jqNv#(A6Wq@WBWBPRs8l z?3~9?RsCiVU&McYmv)MYPn#Qz5FY)R`K_AKDbML&v9SFZCJ0}n7~UniOcNx?UAuS3 zgNsQOWOYs=U>t=w>+gPKx_AKtHxyQJ_ad={&%Yq8B?sS!8YSVqc~1DO`*I$M!2rF0 zpi4q6&)U-kD46E9cXt+}VoT{~uWuWYbJtuiXZ(^X`o;r~Q{_DlI**GLL^Up*5nZJq z*R10QX*Uibfz}j9gw5Xc!9@<&dYFl4hDUmh%)hqHCUe^i;89E5k*&MPAR0sDW-&3i zIBdqKXC#xlr3zS8Kl4$1L`1}w55dvEFvd0EqGVPLBGIm$JEPE(Mi934i$TCfjZ6@i zHN)H_({Sh<#}WRL1E#5mznuall>CZr2*rd<^D+MXbN3X!nYl~u^G~_IoE+{P5J^xo zGq?2o1`%b`>`KL?PY5Lsze?0AY@^pOClInlYv^y9U& zllKVQ18q+Hs){r>$=?lO4mLA5a=(aO0x~IN+?FgINm8C=+P^;;dVnGTPv+iIpc(%w zT`MBm*J9_JINHBtG_~JUkuv9-q`LNO_+Cfki!reo)#u-yUw?paZ9;na zHo3K*bhv1}u{%dR>FI|<%RMX0jb_f->0gtTmF@8hZp3iF6_RfTUh_&de9Se=I!wriS+u0K`vkq#TRqZ#eM;%R zPWg!Zpb4MEI=!Zo6%+DGNus6u*0y$d3Yw2On4i9-xjQ?Hk-0m)c~GRFRKwK1P%Nqa zBePvD@s{ufdK1+kLATp^t1B+Ny@D5*3W;Jz|`$X_V@WB&1oy>w~c`LNNm zALTo#qjRM>vRLD4-f8~zuR!<81>rwDkO5QvoK;xxLD*|LgD2-A=}}4!8M->z))`WvAcr7u&z~{4Nq-tc-lL zrA1neT0Qjn`X5E@u5M?nqxpID3S)?^Kz?3P$UK1m`SSrQjP6zH!CUirhib~35?mri zA|oqTd58W3#bdu>szYC~Y_Z%fzr*wDk)PaLA3JxKT{U6SXp?x-QiIqz!bc>J0n%Gu z&S$keZ#~vks)Me(ol>FNLZxk4YMU}$hfjx8JvO1p>|N7DNaWuMi!IF($fS;c< z9Uj$;$P8|DE_@M;j4NP?@KVG`gdMI5ao9nInGACf*X&I>;2X(&rWKBhUIiRa}z3v>;(Av$)B=->jP6gw!NK8 zs*`jDxJ*+t`|4G{el@ANpKtg6Z#wO}v@Qiz?z=XiNn&&QieVZ1cWFg?N158mH9S7# zHJGi=|D}c6xWD2xWCI$|7y>VUVPp!!Y1SmLPG%H|iT?Ss9%=G0S@Pj9ydalQ@B{*i ztdDMkizUOAY7te&ObVIZhb*E1jR3hrKhTvZxDhb-WDshUmX@wVe@@Bqz@#(2PZZ=k zPT0}J2Z!uH=179Hou&LBjH8%I(AhcihU%vWsD=%K)mGW_tC zJ6tA;{bIoIaZ^Os=q*{ZSE<#3SBeN%}nHA@UDldy(W2e_j z^jKTHU#ih2SqlZqZx^Pe(m3FeE)psi5nZ(=N6jr`mjl%^^l20Zl6+U?Bo^)F@SQy zA3&HDaoEY0T3a3v7qRNTja?E&E><`^`Te&Ka$5*8Z}&bbcTUc@)A*+9Tb9|E-HrJT zl=;p&h@DoTcMNN^KY7WSEf8T6DS6Om=?+PJs1%M4>D~{|e3~IcbYkpPDl7PJv(}f1 zWcsXlw$(I5(AW=+TJcs2@p}=arvYhHGzI1A{PX=g-yZSD_b*J;wH^B~FI$=}&fAyf zQte~DFqu)l^WH`oOXU>R%m%l!aLe=^bl$+}GeH6j`QJKZv`t_T;S9OW_jrb!wvA&W zx8k5EkT6|ceLW2tR-$sMI>C0LdqmgY@pdeH4XUu zJX`~>!@R^MaL3BJ4NzS`u0Da3v4xeKz(9keW<-)QW~H()ULv`vUdN-+anSd z<+*WRlaZE|pBwM6N5|-}E4MneoSnwo{OQxvR zGp_#uPvLW7{I_Rwz2~2p&D${~EH``PQdcqW)QSlPvazl_KP887tYW5HI2<`#BVyKL zD1*H6_Hug)rdp1|Il)HHN-ilL_*F=VI-aB7b2d+*c0d&Ox2zy3?O$u;RuzqL>c$jz z_$h1?+MZy4mDSZ0qg+@mWTcwJQz#g!xbNwChPjJ!8@*MtZ&=AZ^?|+m;H}RSQ>X0} zZa7(7EpaeA6*pTflY1ocFa+6yQbaVyG==%<#mG}6e1=Tp+p*>4WfCj%Y->4;f{y@J zJ%|`Nn$_W=ZC`R9!0DBpf1HLxos3w>{c`A>w0FN;8w58FIez2L>Ozf&-@$b}&~4dW z2#9uTc)MpC^{2w~7u(zhr*1LV!d^Qd?%p2d3bV<%_u&!Z*C6Js#+;jDUrFqKim8f} zO3AqrjBAilFd{aRSZXr<61g#0nb+p_A3B~j~xgZPH-tbg^6|dAIqMQ3k(>9JJM|cYmaGExY ziEX1w|Dz`(VKY}^U{+c5=A$agq&}w@!p9b-KYMRSEu@R2E zOhFd+h1U}*GBSlzi?9vAkDE4aTR)bv4~QQMA{xX7epX`=29!WD-FT|!drDl=WdAzI zj0#h=$jxIJ?wL$7&PU)>Xm-{s5u*aV-w4l!poFqXoIYhgf~bOgOnrMlmQvLI+_-sX z0;Xi3Lq1>ne&$y+r>`fwbyFhKs#)Z;!>%xu=cI{^&Q5OIba%bn*Fk2ZVI)aatF-VH z$?Rfw^H~77Fo~SOx8mS(RztQ@moGz*i_hn_6UQJi5@-DcW`|)o4W3V{V(y;fTS7f}If<`k4 zbEROSWXY?eq+AOtq*7H~O~p~#B3|l(3!L->S49U}2`egn*nK<6%bY6Fmc#tyM5_A@ zCub9%?c*Y;BNmVBT(**}4tppa&sZBXV|8mw9&0rw#emIfVfq8dfJ{y5VjfC3VRBh* zfk>_ZDd}G)5n>~GCTTx%Efjin3$re44?;84T3|7g^!pgpMlk$LvDJB9ZwJzx-}|yo z3@Us>F1*4g(xN}=hg&82ZV8r(H3_o%qEbAs-x*zWqxCFRBfTE^Ij=&Oww8~DcvQ!Qs#M?;_Gp(e^iREl679BCq5=>nt zg5@MY@AgY`eZb*l=v3?X(c_QZ+P4T{RzW}zCSm~G2k;%@MLMJDitf#lTwlE><5wa+ zeQ~a-%1bU$TZjwraArHtDMjKvM7@MCYl6LiOORwS&;b2~ihGzy`yNL8Mnjj3ErY-m z>>OcSz8HHBVQQ^0wxbZ|qEfJC1nUvK3VPl!1Rr1{`QLy4Q$5El5%Uaf;~wX}t<;=I zkkf_4-M-HHqwULP>5s>@OFT6J2`IoV89kYYEod@+1{TJmmvmC+rNrVbj&WDj2o2_LlWch5*wXxL?>7+U z!HS23uZmbZrJzd_)sh91bhzp%k9{cX*5pm8SRHLeKcblszbMIE?KA`Udd{;QDvF@R z)G74)(?S;ijD4WL6YCqd1UmP8xNt3FZdoT4(2?9v#+C@kNkU1R7+6wpXotM{NZLO$(Hz!)){Y2f*hxi3@#=9LJt^MS<4|(R zLuq?T&&+(gK<5QZX#Ew;Z6rr>$8|iPh0gCK;!!NE;qP`vG~-9@cpjNnKBsq z6{-|2dl?-=Adj^_IKRvXR?l$$2`D1ebXR-Yc^3^hk{E+p0li}2RwbMFuFAtN8(K%+N1*wcR3P!!5G~)2{B`t zUb~Zy`0zv(p-$cT;X0Iqs-4Cv_yEr!M;t{8U&^mMaJN0Zcqvz#5Sd-BdXgtO#MHBq z?i4pXx?~5IhOZ0*oX(@a3AxP*l&zY2LGYHuA1j7$Iaxf`p!f2!eXgp1x>godU-#Sk z)A6d<^_Pe-&sr$RhhZXy16`BW5>6gfj-8^#yS(8ci0=Dux+$(vngY*Vj4(11AcR-her3)?>q z0%P=%yy>(48*!BlDRm?u ztf{IkpvTRPR@w9K^k@-ZDe`>U$8tLVP^j2cemOY`WW98h$E#9X)ov z1KtRm6dCL*NXJO`O<$GN;pJhSuO+zG<~R8NwHRt36W=6XR~OI{2E5RS!~;?6?|YsR ze-fR-|7^4Ti`#cexa6VCj~S`gYtBnn?-Rc)sv#2bt}V;*iNR`L(bl+MyXYtLQ%=Tr z#YR>(y*H@7&4W0wUz%CUwpZ5wMkev2QEJL|@!<(%e_m z)q(Y=1--ndf4#*x7S;*>^m!%%<>t$*OhysPWhDxl0tiDPZ!+p3rTCVqJ};4@Xn` z7M?mk@!khNPoWC8874e9>UOM`%(pkF@h$(B?YItHvG=wJJ+w~LekVMPiM$jXM2CB7 z%~>_$v7cJyUiliwR}!s}WP^$NXS~;W6LsxJ!0;$cYEnh1482Z}Ygs3@A#20!SnsIH zXXa8K^nXFYTqA4S@pImn!i)k7&xIS2-d(;QjrC1$+W@dbr}#kD+3ejjFX1HR^$vsv37zTUnTgr&c zI)+DBcZrFB%rr2%$8s*eN9SX7s*|dUFUh;?4)?8(e60L?OZWwe*gW?n3guHCoKSJi z@HSG*?jfBGJgilJG$K*81JK;>iDIFqK!0r~(_3-xQ_surz%LKp4Pa}S{P8<4nUay) zGhKAr=mh>88&4_6wksH{9v$CksS-6+UtJ1=!~vhWl^;kV9s4so&s=Z<=ZgkCPXOcX zQB0f8kt|%}>@snF;V9MilvM9y+d&+kp3Y-_f0QMpP-&Zfk~<@-i5F)Eg-dB=J>E9q z+uQu2rlkGD%k#S%U)uJ2RaGz_=Y^9e@bQt}Gl)V~$gW&QsT#uIfvTtA4d^aRuFf-t zhwkEIn zvC>|mQ7AG@_bPQZYNKvux^q>%>-Eob$IA!b<8Txu#4c|JDJ$#!SZSH3I7%^{DY=xG z`^53&?nAriJ9LpO%%5Jcy}1q*Imdy(m z@_yA-p?Tv9B~2#;#;d5Rg7-Xx;%?|gp*ATZ)$2EeG=0AF>Mw+wJ$;SP#Ru>^J)((X z*nwmKL}okAo)OSCf!OB6@#9U9;;1habDH-fVk7k1Ya?joXctGdr}fKSkK5)jVeGMbH3ffc5-2*#fGCk;2fW?Z{i<3k{R5lmp*ASH*I_}uim7AQ+L}_lYwaWzT1hm zE;)QXUy-H@yp?8<*+9j)ad$Ga_rQFG8%zTY=%S4`US6%BCPPSMGn^Fze=k8{HFzEkWCYPS4YWY0lPJDfkjPS&)RtNxRRS< zWGCe*=9*{kY+BcyKKcgRneGhj-q9B_59bfXW43G^g#gq>Ak&Yq4WR~yim$)R*p%;Kb3z8d~7e?TLjG~!IkOCRPD`F+}UjL2Pkzz&!0TwWn1VQCC3+jyz+tb*JpHO03tT=SO zdT{wa-UDNmg$m3yu=5|jl*VW~sCmPi@72_r~m6kh$~5nlw8+^_ReCg>s8x z0CPAZU4$9|kTGMTq3WbYz5^Q`Vp+v409`6VwzK$Rn+g$QbtnM`#XLbs(-zH` zHH>sF0a+^uayS8BAN_=5T|YquRBB)kV}fp)r$Wi3KW3ho1{auQ%8ARg%<~0+99o*{ zYMIE$wBBE2S9YVg_rm_iv-!dYlJMq(cNl&&DhwUt4>13tx$}sj^9NV1Orgrl>vT6C zz*8ogT2rhX(YR-E|9MbFdwZ&+OZ`LkKO5DfkJVDTHqPt~l$gu&{b;M&H7fN~FMX4D zIK4$C-R4o;c+2B#HkHB!$*@He58A={+d|8pNM02C8Eu9-j#^v zK%Zs|M1U0F?f}~QqI~!k9wE(THa0dqAJm?K5Zkt5Vru(U0|Vpefpm9d$?ga!Odh{$ z9Mql~JBV{zy>8vOu{>p-4ip3)ma>#EL)4KouBtgnJahBf?QT)BOQq7`{yXwV25@}7 z@78_>k2_y%Q*VCotN+!S1FaJ0`j0eAn*gVgp~TU_H;`P3V*n;JXsrne0O~%|APE`H zNrZU^ls+bPY7!CD&u%n8+YQ(>pkK}58N*vkil8~fDBL*Ou1BC3ryNc+ro*0SBX62A z;FzDzG^`zqPaA^}zIhe)Imp7DbFJcLDCv4=D)}iYI7-BUX)j*1E83p#Vi~6!57-%> zD-M`S3JH(v=A)AFUbZWEZ1O;DSNioLwshCB1@bp{p!><`LkDWVP@(~hA!H@KG;8bK z0qx3AdxF9*I{>nvDIKR{Lx=9tKhsGG=L<(o_sMecY7x_RA5-}rki`0Wj>S8HhJ(yC z(N_MGO5jCCiAuKmY})}BUDiO}y>}>}5&%)`xg?1J)86w($wwwunqRv%fs0)!**aLM(gR-?PMuINt1Kwb zWFlk3rJdX$FnfDD)scH(@ZFqARZ~`k6uB4&<04nLN>!3e>}tzx#zJ8_pS{pn+COwR z2<2+MGTKWIzxgcDa7h9TUGESJaM<=3oy05;v1E6QJiIG#MDY@{E+BG*=owcbTtG^T ziD{9e-$|h!(3YUH)a*dwz3lF{=l@>14*M7>l+Ti-tvL8$( z%*RaW2>W6bMxUy(FWnp-9y#w%@*wAf<-r_SG|{~YG@Iegdp-_DWNnRai?IT-4t3CT zM;LNc$yG?3St~hBu?NrGub;>{5!p%6Bmiv5tv2;w$62(ut{hO9ocNs)>O%BMqDG1@ z!lR8;we?i$Sel+qqVY;bbb01#G+|7%K0~qWXS(u|vm#jg)eQEFmJafY1+Fedp3U=D zmww-HJEF!CzRNR`?vJKglRTi6LqwogVB=LXEl8SU^+Z0#YH8aJfS~XLp@s}F0+43N zDN^?PImYLA9^$P-=gm^|AvEF;f(HB#g;5@=!Zm;ldT-csp|%v`HE8s60k-?(6;8jq z^uT==)z)PF;;vJ>lXNbLpM%kOVKCr?X|oWt-{{UTMZGhW(@dhOSu15>`1@Jh z;3aJ8elRiU8kQv;u_0G*a9TDJ9V%l67`!eb9CzSR+zVFi- z6T)u#E_~+SyN*GEoxNide7G^jjCaNW#V6TEs_efrz*L^Iu|4TG{9v!hfwTKBf0T(u zpA^>2%1rkC{n5#kv>ns*=sF|h455c3X!=^eiM^at7O8$LE!*w9^~A{UeB}N+@g>II z;47cB&2M!pgmq&4s#D2DgC5&Ps&PRoU0=Tw#%R=8i+j-~sN9yfA$1G%Bgrg(%-bT- z?5DC(B0fTK^&am zXC)LkIGcSc;dksM?kw?in#l3+-s2EtoNO&9!vAZym0sp|>aj%m<-_fsC|w#tYF#djDWrSsMl zYI+uNk&Qe6odHjit^&#MVUmgWH@jK8d zlm*TO!tMb~sMDtYq47yTlKz>duJPDQUFYZLL5u-W?4hgUF1v&Zv48~-#Sj8~51Qva zZ8N}5=LW7mNIN5CU1&-3u%~8CQ^?G@f1y20c#UoRv#kXC4jDm{UV6E?lLiJbiF!GG z-@;we+|-r>{g~HAw+a(J*lo)M1_Y-m2{TKtuQ6pUOsR!^OpuLCRCw3UXg3wm^yQDW zy)ZOYXnOuHx6$OkPD?h=xbsUKb%O}Z}A3X+4+9`Ngoy@h-vB1!@|BLPa(wEcq1H1}vxa;11KIoEyz3#93T z)+mT}wd(aj#*79Ix%ohEJq6l0NL7oJ*FOE5j);&D2u}B3dY43Gw;c{TK`12!Gn{R3 z9u^90u_owgSlr@tbU?&5eS$DxgrHs(;@Tr!SLhD9RqgT;>HA7mk)AZR4UJlX0D8_? zg8hN*`2lZ|gV7w~eUy~~F03YH`ar&qOa8EBgi5uDxXF1k^{PvE@}abvMvDtrSEH7c z7ws&o--$|+Bk)lYP{WR=Jj}7ITpjHnZIpR?rwZZ^r~oH|I;L+ zj?%OGwzj#zX2jOO0lNXc*A7U=5sYaJBrTaakIQ9;Ly&C%yB1ZULE@Yj)j$xZ0{V~j z%ZTs*+Yu>UWi?KTBTf^@4$8x8Kr?M#cN06Yv_lD^Iu&7->;YrvgId3EnIx!HD^5y; z^OvNX!E_ES2VmTR6q14GI_h}Cc>(IBxxX?Kz}GoESw$Lmpcp_WiueNnJ12x{44}La zO9sggA@(-lqzpULweUl{Gn0DhaDkW;r=;N^)a+OH1o(;8zUgUYj2H_*>Ta-e z5SkV(>L!?Q|EZ&W0ZZ4%`sCnCE3hGD1j(%*P*|-K)82w1IsE3j>SuoQc z2Y_%IO#M*vg?B88n61!uj)Cs$1JL>S>3aO!)J)Z5)^R}OH6zIc!~=vjXEU_IAYStTIfl>lnp zQh=eg)%}2Ea{w}_38G8TWq*YPt&rp##DM5^!aRXQ$a4FDTm=1hO9w}surtzaÌ z$MyZfl0Fa%Cd3lw| z*u}@u&|&0K{!A^JZ&!Q#w8#i47`-NdnOH+TOdNw4bX(9Fv(1ARJn3!cXTPROU{#v* z`oJuYLx(5`6d|GC8iKdE!vvz>nE%wFl!8H@iSp^#MI?e0vjzPO*=EhG!)cr_lnRPC zMD2^dn1-Em4|ug~x;g;0WC+wROzNNKQX%=AvqMRK8<J*-d~|#l!hWq~w+@q;U*D zf|vbHE9!{epu7W_1}aklbP&mw!$hCrIb|q!pvhO@c|ZnA9w}SY_8gk?!lu?-qYsFC zDm$v;+hs-u7{sL-<;%zA11d-2_bOwyESpZ#r9ONl_}JxnuH@$ndozXZIAD?ZcQ&oi zpdolSG2LO4P>TzoK?InId4zC*x7GnNWPp)B7WbR^(f`bXM8g{vDjnvc0rZU&=fL!~ zz&dI@c?K{P&|^>xII-l4X(;1B>ab-+mjdL8eK!tAFe0=yR1B~a6k#Qcx?yCX5{BJ= z?h%ORwfwdR=r3N!^=;50d)QC;T>l~8D`^Z%I3qt4Eix3B$$HPu_PN~4osiY!FC8(b zs+3EK2@w!&PMaUN%C-R-OuMn}9Fyb6S4UEpXtbbKfJot@%~0;8FmnN*bh&e?EllP@ z^$Xy@BT`r>sN_M19|ix2-U?(l_1%&iZ+$ycIUvT2fzgS!E}(@5y?=V}?c@?N&3D6` zve(^yy2H+=?<#Qt9V+bE0$Rs$I1t&!TU|g_hn!X%g9P@oGyqtN72gCs4sgnDPQb2z zKB^pg$3FFs3vQ*bTG#8j!6&CCJDX{r_z6qvx^VcU#DoTKcZ)Q=pQQ^wH|^VD;hky4 zmOVbGG|{zbm{tM8sPHpr_U_6tyochBXI0Mu*9gE3JpCdpZo*5 zKYlbm*RH94j$XVbiU!2NdyW9~FtqPM8(0dt2Y~#66w*YsMW4sv2v08nV$svnKOOc{ zLQ@Wkx(P#7$$%W5r2%rGH1wA*3!=eXfFxCKhE-U7t#)O7D)3I{mBd`Hqpub`r+D&d zx9x8*^Ip!?7tDglca3~9y4W31T$9UAkpKtsZ5kW$p2IExj@aL(MMfKbt}J;|t4u4u z5B};C8H<*S9BVmCCr-=**UbxayA}`dEj3kCvxD!+bD z2NbWxZ-+}hcyKMFOgvzlTOV-CtVCbX?GX{__j4z69QjBdX5n92y+kA7y&3fwSgH+@@Fo|D-34* zKyBR&IA1LA9)xM&ScAS`c%La7+s?YJn85St)9ap;jQPhOHla-{fAYhP&zOa*X1rR< zmb24FJ&DfIDo~ntzKw}ClbkGKz4v-`mDFadKIiXb=HbPKs8HXY%^K%8veQnx0%vfT z&10#X0Ci>35X%rGk0vD{coGB*32g}a8?WJon@?TGW1D0FS#^IVQk26cYPd3wT{x1H@mPkmb{F|>e!Qpa!_v7uO z+akZY`kFHQ)P83xjXFy*VauF(T@R}tHUIi^s_^lx7}bdS62gl$c>N$#vCkDp;u}62 z@qZz#%M-K+YBI1M=a2?y>e&q*h$v_hfN6wMGSyM&G0FjdK%9IQN#y}#A8}C;k{2+; zdR3<>xM@Uz}lz8$T>BjIbQ(WW82oU^VX6Q zg01pB{VNRtEb}|4beG6Cf}%ildLJ~`H~^u~43g_eI2vdpaOzb*eP+x*{p!^#Pl_b! zi~83uOueh_U)03F%0T!d4uNwEnCu+j55Yn^Nv%wp0zEk0a##!!7)j)vknEM2CIY$+ zv@|!Hv@%1|+f)Bs@vX+)t~mdAL*A6QW+*#esh(fSi^MLVDhzmEo5 zRJG9>G%Ry~k${bGfJs7ex)&cMLVyQGQ$MeQ8-Ut4gYB}03gCy*ptFInB+k40sn%!oHh$z^#DBvurwxIoUG226?w ztScjvT`nI^;|VCZ?Xb};xe~TTcG~8A6C@;26@5Fptr})6QcgeoXQaBZqiXmG_oZBs zJx??J@DueWhjXEs@uL6B%{9);ST}Q3E$mrdfZX(o8COKw6%%O@hh2=%iGC!mqvBEoVoLrb%t1icU_R#K$am+0e3xduj60H=-AXPn?0m0{WH0M~akyq#MMn)nrzh$w{>^neZMChnzyCUK*+K!b@< zikw!q{q)xjkeEP&#KAct$=qMv<*6p8zgQ(R-!N)CdAGWf-}RW!WJdpV=(}2#3_h|T zT3p)dQRTSu!@ml=hPoY0KB%^peIrA&EZ`^KuSB#wGmP9f8YxMPWPlih6ynKKI{Y~U zeWb=C3MW7r21!%{L1Dx?xgzwOD2$(_Jzie+0{E|tw{GQIjEuK|Jkt`IxPt(|yySr! z@2Ok;a^67LaDobSEyOUu^?K~z>FMFo3^cN_<_r*m{S5OC1kenDeo&l6g$mdT@^G{e zq?^My_1TwgnEF*~(s)NH=hz?@Wb)NxlWCj1Wt{1y&l;BVd_0U4;|dRMF+%2GsnzAb zdJZCU$Xr-_+Rk{ZwaLdY91y#`(=i~UW0ZYz`~$0km@iFA--;#_lF`qgc~Ug?)oYaO zgnoSlJpA{20^FdKUYV*QcVw#-`a`-a%?3bG0?pcZkoVwjZ2)8`aUR_x8JkH}sS{U$ z^49N626+W$`$U-Cy?d^GrrwJa6oDzAiH&8?3|aiyJPkuR;n+7ptqzrO z5Q7QWXdoq%mCgbf$oM=C%-f|pqeX@84#GZb{7 z#W3aN#V)6DIxhWJ{^bdkpClAs)9;`4U-~mR;9dO~HHVN1gOP(tFkMP<^2u&5#*M@k za}=gX12Ef9ISuq`@Rxc(-wR>=0prb_s{tD9?BIcst|$`I0-`sJt&f6wEfW6#2?JYW zs4bi$(5Pa%V^CAyK8&aT=e%sm_uQL89U1ot%((1}C1`NXCz`%ox-({#9Az0??{`wE z|0k4n(q^Xr`rRzw%Prp|>7T`r|1Cy!EwD635R}&cp~lf8C_i3DeV9BT-QeX}m-Wiz z09%2R`5W|B!cUa)G=**7KIkX4Z=0KEA(9ui#Z<-cT$09-g-czYItSSxtbWZFICOQ-jRu6B+-;3~VMLe(aXItL zT)3a>5WyxJ%<*K>u<}u>hR*Fe1OCU$EovY2;rdo5H z=~=P!j($yT+0^(??6JV%Wr@!$!xrdKL*ST7=$%j6J4|0Xg>9H>aw)if$xEc3-L0XYbd);Zhh;=vj?wtl^Xyz_!3$J_U~N`B$K7OHi|?D(iGnt`u2Eo9 zn4&)kUwie`e2^2B0|WW)b#a~}#^;w;fMZ)Y?aj_;WpFei=djkZE>QwbGX*)nL~NfX zO85TGBo`I`qUXDizKQ*N#=+Tnjjq?(R_isQ7p625gJ+wQa*i^RRDmJTZ_9br*V1zF zA#pg{v)u};?wZV^imUO8(^SIyy{qFWp$-+Lo()y`A068qjteINd}`Xi41SeL(o)nU z)k5wxBeXdvXg@~G&F8vuiPbXk+}PbxI06m`56jC04(YI#hZ6#7n9An{kfi+2ZcXvr z6OC-BN|isnheeSA+UT>RU-^3g{=WTX=`d;4}MF^N{xCW zhQE*Jtl>j_G%e`=LLW_bD*S-**A9^s(soux9#G2>X$Z68=2SYKIF&Y%F>-^UQHh_f zJCF$4kTQW{CVR~)0S823;jN>iRwjD+t-gj_MOz-D9mV|7OVy5uhIw~Nr!_~PFay24g0A&T3>CPh7J1Fz>?Tb>li4k>adB#1qC&=NaD(tNqz z18`73Udil#5N*Wshg~&uPDfNnC(Cb~duAGGRQu@qwH{Km%wy;e|`^7^Rl>Zk8;0-|L^5KBn`!#(_;g))nOG zxa*RcR(7)N2IgfH)^@4}uzMQlYE%l7a ziNsF7cet+jvggF^KF1NIzF1PXxjiT{Xh}!#qpmk_?kPW|121d1RYM6?P5qw_4;n^& zg=EgX2cp3yJ+5_)e4%j`=~(U`egFrYAEJBbgWS$uxVjx=d=ujCzHX zWz>_13fB0mn*vxXgL=FwO?YYvgG(nW1fus28%`Db30Is)ALi~2rE)(|_fE7m=ZQ47 zH)kV#HcGI3;Ztnq;ed;ZnDg`-lsenuR;BVX;~%D@_glf_B-jtXvLg_a)&2T`Xa&C? zN|S+IUzMn>Z%vW%7Jj7fk6k_b%{VVkicP7EG(LN8)doM^dqIH1bT(`-M!?<`u4FbK zEAvasU0F{tjF(vh|ebRi}t!PqVAfpHu_1sMXBV1ZxQw0srk(H8SRTFmAUVyqy(q#qX!l) z3^=HzXhDp7?A_Sa*Db5wbvhrCl1jE#FIWNqRMXR`gS9cEzuXO~H%UFJs7 zs@JNnkQ&q+dAU&dnXfpXY8K zva>&0|9NQwXweD7I@<}2XGRIQ$!$+j_Ei)5)Klqu)T^xcGZ>2<`tZLlR$ekioqY9{ z$3ZQnYe&kb2ASa6l?GP9O*U4Htz5(R1d}5Rl^3%HPRpzAUB*HJyVfTbGhfXasCwDn zmz4Hgy&z~Z@6HTom1m-NlRib~Wp@l6iFSn$(ZqQ1H+H-A#Ph@R_6Pk)_r8PO5uJ;> zTE`7B^?Gjo)T@4_kAQ0bNT-6olPNkuDWRiH|?G6Q(OGzp6+(}eb}r2yU4qj)=u?Z zcJJVbaZ5~M;`=@MINFh(K<|k8i|gDhcT0S3&JZW3JYjsgk#NE)|KvPTa9PXp?_`14 zeSkShtru7X|6h1n-NvPId{Fc(zogM$R2NkAk$%c@-V8Y9~%K-;|6k zu*%ww@|eH~#Owzg``7R4CC$zpyxMqC!ra*7h*hx6>u;%Fqj&;;7kHMprnKD`OwQEv z8PO{t`}BQp;Xji^p@Xuh;3*?oGd3|yUOrVERjBy+u@`3^iTW+mD5;rwI=N@-69Ipg z^Z(r}>}u$JX_p$^U=V#(-K8=y;LyMO6x}~HZa$_bP}}c_NQWN#<-TVrb;*Osj*{rqn+mhV znf;8{unYRVm`RsVY-ER%x$j3uULU-ObNE0Dtd*aeNZw?cj!~rSP~@qM3bNez*xZ@; z)bm&Qj>gh+e${<5oXqT7lI{I8o6`7=Qk(yHUb5Z6e;1JqdlTf(jz-WZ5L5Qy-wm$U z>lRYhP>=Ep&ZBBuI#}^5-FR+HDt7f{0vUUEElW8`l7@c|MX%6{@bdURQR|`MxC*D{O9wAlv5d-eU8}$>8tUJc)oikfG_1!o=TWefP0Ll4%`-P z5$@g>4c-TH{ha#DuW9nTnt3FrURdE`_rhwfyRxabnz-Juy(t^F%hoZ56sD8%Pxls) zU;*vM_iz3xQZpIceQ}+S9nTfxd!t=ExG90Gm7%ogov~x#MGZNn2(HO~eEI#DW$$Aa zv>Ud^ehspiN@mne{k-~*^Y+imm0k<>tKp3H4$`nh@E3BDJ`K5@rEt#h^n zS1Ru%I!+P_n%MN?`}rzSE^!?C@0)MUs;6pYO%1+Snq-pYKkQ5q|99`ssiPqXg69qU zbft^JU0KeSKn(m4rBAlSmj4xXHfdJj;2tlQkX#u{@u=ddOCi_@WZ%qQ^g;Xhc+q8v zGjCy;Ysz8zHbrw=Etl)XW;Tmwc1^*SovTX9AzDm~F8-ild+L$<{^A8;k1|(& zy@9&WTca8~s--1tG=i6TH%b5s_ero%^sXKXvl4$Nnu? z?%sm^v53~}gAw(5a>5<(!W|U670vpc(Kn5C;x3UG)s^K|rqE_t`jU1CIdzBXk|!lK zPvw{&W)<4|B3v$Xn%=-^BBpeDA-&MA$IjN;<0aJxssnA4QZ1cz9BGLS>O(8Vdmo^d z$FBcxu^Z+qHiYOR1f7q=$S<}$|fD=SduQ}vNiy?gDO@{=`1=KjAz)N*@K z-1GN$(H?liSpBdr$>=wJc_~Hn?$PNA2B#Wi-6!w_1#^iX=}R9?9PKjUOe?~TOM7~Q z;Gc7IA4RiwzMnZ0lb$axm`tw^PCmaSVjDW0+C8?@G8$S>KBki07c9=k`hZ>=Gm3#> zF(Eg)3zeKxpOPeb&L6(Yv;XkS=;42mKf~9usm5u*P-U9g#*5_5ZQ?JkyTUGRetw4Hj|J&Ku3}wJ@?tJCmw!o~stkVpEqaHre`A&PYz?M&!+Ep=26tvJ zkvFfg3W?B=Ra%}PHX3ShI8eRs3GW_2M~Rg)xIWu6-Kay+VZtcbx=)JvqwiTV-8amt zfXC;ygRNljpRM4=bJ<3AC0JspMX~h4oBd#ung(k^|A=ws9$LybR>*Nw`1n3H58{>5^CpFhId zy7!D-4@L=Wmn`wMYz;yMv2GSwhAC6FpbLG9hqyH!U9fUA;xMi^K2W{yWnNOC=em0= z?}A@+O?k)_&6}6|V2d#RJ>hyt*!n$K2RyZ1>p&wK^|a4Wm2EGxyzjhk9O|&fVWIR@ z`nX$hDTOM1xuHo}V4z>epUI=D*!^K@{HCovKj(3K|0SaU;%{yI4)3*!voi(oIGxX? zlkdG_YGv$E|C1x2-1(O{>Wr4u(rjxtK~G`QJqNc6$x_?Tx?c0mde#&DoKfgymmqVy z0PmWzYw8SxgMWp1Pt5aLqc7ajGPc2KdXn%_kD_9$Yj)0gZPd}md?CS-St|GaN-EAnF^nm>bYtc zs)k{YtYXgW32~o~;nGZuZ@pP`V}BJ0v8ymFHV?1yj?~ArV8U}Tf3qJ8nzW5jFr>sA zE#w?x#u^K<9i}~Ffgz;wi@g)d!s+G6YgYeo%jiEl17(vhTfg$)(k5j7yvei0T1eXC zRmexl+?!rO*MXl6d6l|rpd4qb3;%k4k;G6|M`^@Fw`JMLz7zZEftHj$jRzOcsf9GF zAOtqn-p=6J-5D!0Z>D^MBBTP}@1 zY9>a;764@=>xVg1vgd;K8BAtL0$3;aO#TosmgwpO*{WVL9iX1Ogy|OJ48Ay$L>v{W zc7>jnvxYz|Tg%gE^6ujK8?2R|db}ELf36?<^6oqXKWqKF>6-Hg4$Ez|`OMb!8B8W{i4mloVA`+8lbEEVl$E`L9bpdCMZmWTzgnVC6|GDkYLH{I|F*+YC~? zLl~<4^*UFJ=ZeJeX~r?)EH}RIz)&pc21<$nD-3b6;F&2vjf;VWr5*2D@gt!`90H4D zc_b;!11a4eeEbEuE1Xdd`~|k3xlj(6|GchG)Y2zu**J{7!e!?WCzfD}O>LrA@S%Qn?cST8m$JCFNkCQ2^?DWJ| z3gxyAHUyAV@+}ykjN{*x!;5P>Zqiq%fN3qv$?)zk%{tn6YshQ86MFWv)jw^s6Z9HF z5WGWO~;|4*3roD5RJWy!|~iZ5a6g^Fv+A68`ZrB-*$bli|UBlSAbXmUa)P$fc* zT4M73=jHM0ntFz0A3s6`gK3QE1V_F%jiGfK563k}j8%2VAG-HV3QFoRAU9zIp!8G3 zTlgdM_B8133nvnj{=&#FBY@gX%eX>B29UuR*p_vj#-~7AhZ$P6T}*`*D2lu z7{3)jSQ3*1E;~*LK-Ap7c)=WSKWOR{VyFW+VF`ek_c2JF5>3V#ojh>(X4;XvR5{@A zypVO4He=&{bnh5fNM&9wk=3HSM&ZtRap9%H7VOC!Ue4bN`{!^Tg2M@bCx4nPkUMM# zsvnx+%eUOAgNe>+;I6%X6NX=7T2n&{hQo+JrUf;skWkiSe!X4`Oq!!LG%o_t$bg^` ziJU0mZP!R4RjAGpYPH zImyO&K6%SUe~Tw1>AS&D2&(R6Hu{7g=NM0U? zv)MTA@qz71H31yYKK!za>Lok@{n1|lF^P3=j3RS>QU`PzWlT*| zXcD0Pc1lBMpcAmmr%c6y3n>K*Zb3F=^Q5#lrwweKrHOB0t{CTcNJdhdPYi@Dk>2k+ z7E9041m%HLMtSF%?JS}PQ_io-$t$@SIbGtWD^w0tpV_;+lwEeg9w7K=D9Du(VVA|J{V-1wdQXN1QS+l_6PE8z13 zmT+Dw*=v)@B|lYyy?^_&To8S(y@kPbLBC$!AYsoI5#4aJ+F(C$@ z0)2(2ueYdXdV33UjX(YJQ&uh&F*YJfC+>@fo7Zn8U|{BR{{lK z`_FQVfjl$$f-rYFmK^_eKiCuyvT2wsK4iBHayuOMC5NiuR6onzw!l}C=vDHS5EC={ z%W>cn#SDq1?O2TIv5F}oH<+0KFkC_EU`n zKdpI1&>#F#CYlR(2Ds%C7FepR9FJrwk%#~QhN413$dNncUTT#?Qr4au=ad>*Ji!b^5p3yA88ckJ&okGoxC#abDDJzRcjd9T# z_x`(mcXp&US0w)-OQ^$0V9>26kw}~ZMqEyyahf3Gh-OM7-O56EA|&h=YTYb~)CjYI zfS%OM$K+EL)nis9nO3~uF=bnXAoS=2ynfgMrg(ih97Hq?Ec#0`0C+*#U}!S&6er>$ z(-oEFxvPT7iC^rFKmdX1Zy>C#>z-U0K`c;EVRQ8m30IS2KlvifKnpuLNo~nleE0m&s&-PS;| zU0}1?62!aJ$cx8ggV~?_{=Uw34DP+=TPl$GL1XtDjVN+guR8i~a)MS$cXWB(H6D)X|fjOnWe9TT&icd2nj4PU~F1pT#Ag zB|K983J@v6Xsko&&bA}WT0~gZR*>dQ0xUQRZvb-?0*dr8Akm=?lo|h#3ELtxff^LM z`V+6=Q<{wqzE~zY3vwd~z`O~p1aS%-U0q#KOlsi-LwFk8CB7NkL_(5`&Ae^6dYNVw zyBiPG#Pn=64rK|7tQ>%%j0gL7%MK!j+j;rhHrDqd-EFs`45z^vto;*jrr(t&#yyRA zXg~!~Lf9wJtAIlN=){X-l+xg;avWF85ckV}cm!?_cqrp&{M_zw+`7!7jCMX6??^NL z5|8C4_@>mvL>lQ6EhN%CeNpx!}AMv7;e9ec|Ut2G)fGX0omUA&q`H6ma3u98h` z@q}kiPW*TMmGqb2v^oZ5LM7(w5-QbQFFS5mv+HXt)A%V(FDva=As>-|(=RCSzAaK~ zmWOCX6Chd&s@oA@8zx|uj)5gUAp3l4hZA>xRCjF%u0SwG`rBpwni}1*tWUenDJuxTmeksLoE!Hm7YFMX z4*p7r3rmD8E8MsP@Yo48dl1l&foNU-qg8xzvb=(V!d4Z(jg5`Fy`-eKR%q4Gjc3k- z56!7gg$8~K3rmIVst#LgeWX$kStCuOHuOIyFE=1|K1?nApLDkx>E>H{aXt1^6cyU5a^zI2eU9H zs_DX8A$+9wsk$6o-##|Eu8oaaInU2B;5n&LlQ-w2(6^IgOHZ8*HR}4Cy1sQ#dp5QR zUhgRB35{s&Wa%!@cXCQjxBlpFzO4w+)wy)+KxnXGUj~>4?%a9=TG$}F4I#n&MLZY8 z&S5EC)oyET9RvTeY={q(^4BJ(3TKA~QHj*Jwa|$Q|8 z-llr{s>iFnfKRnp%k#mkmRtNMDBJ4wUQ~f;O21XO+&ky~8WaBVy79f;EI5`~h;cLQ zpFG|JE(;cR0*s$5VDga_MpNtE=h1LkIEJN(Nl7;K935kjK|X#>{#*ycSYeN^rLGWY zRBOBTmy%>04x$N*sc@AwX9a7V$@SuQCw)57{~Ajy#-=^3)->EN^?gy%0Gt8XQ>1K! zP6TA25$82~dm5&ZHWProliwpz zzbI9Dt)Rla^D2~GM$>i#V*K1=@s8s_O zFx6Uh1Jq*?s?^r;XEYwpfl;VR4fY(=>QJl}^SD9WM4Z|v7uqx3vZ4GHb&%qvV(mxd zhgiz}=liT5_OzVJ2r4{*8poRZ{^Ys9@kEav!?@5`lq6&R!bH8q$jG~}nKDqE!9|Ym znRN)qxBydMwDGLu)la=F!~04@Do!zeZK|Pp-{Mv>JmBEt#hJ^ihi2e#qn?KY^gJTD z?NSCT_lp*Aje`+1>Naf(NMD@>`&#%n=L9`o;ZQFEuufiOY-2fGu<)rNTTtle;)ZgB z!iIg$tiQ>dz0}lQxslUFx^{KOy)!klUe$jiXTWN|{_J9PbU_b>D2TA{z>scf5GwA- zXl|CJdVCls;&OQpe)&uC@PW7pE8H|PzRuB=%)6D?gU0*dy4A$dCDPtl@U(gz@nJ5_ zbc-|B>w%JGi>?5vI-w5omp5@jppQ*aycs{X1wrCwk^}l%;mbeJ)I7<+>EJ~~_FQ#ZF>x=R z?=?ROh|Wduf9p0HpV}I|+1^%|W3Tv5iZ@Elsg8&^>0F_DS^{G@TXJ~D1nt&*wB14f z{j6tIRRGTwpuT4N7Y!BTWV-&H0q-Vbq0Q;=e!9ygp%DUCr%lx7LFY!@et+eJO z1cz@`b5szeB+JF>EsFfte0VoTO;_)ao^qDW_>gd*dQXW9=Na>7X5j-kl0#yK<{oQN z^#;OE)#RAi7#nNrezrOZ1pUTbI+f9D<5tti%=uk*s4)IPQP#A| z2elj=3J{DMM$=D1P}2^i90d&xGaFSL0zz=0CtQzk@s~BePo0*44%~$5g|xmITx2|Go1LQKe~2=h2ELVt=-0qoF9iA%kf9lp zS`OogI}F_UG^wlC5?v{jWK~NeIJMII%ui8qV^c%1`a(+=q1vveb5~x;ZW0q8(|tn} zVBYpyQgfAKzmzo>0EgNc2G$~SHyRDu*4`deX>4q~RB>L@05P$^PBZ<0dg!Ya3fJwf zu7D3_fFv4SHmG)Ib}k4YQvKh)3OYjG698G+!sfI8T<1Il1W5A(w9ge}WYFODG30(S zb!)R>)*Q@05l53v5Yrd_DjrTb5G4{Ut(?l#j^f;X!N#D`@WWoPjtXkB^ihF%2%Lk8 zu>IX$l)JqnQf)HZy0+L*yiI?yJ%hFC3Fg-YW~MS+yQTyz!yHgxMT*N4J$Y#Wfd$>} z&oDD!0!^w%QtKcrfI^}Hn6lkwc<8zAC0Q@GqC|DI;nXFk$wsfgsQCe9g;|}Ly;DS# zx_640E-|UnS(kmy4pVpCyY}4=aBXDeF@GW3Oc3%UKGk3hJS@Z${zi&(i#V{`3kwfu z8(kUiW)OPY^ufKee90lvs)$Va%+pBiORf5!G6+<6eN1e@WygM&;QH0v0*j$I2A0MZ z>YT2>G>}bd4`M zs?h6r1#m{m)tL%^)ZvbI*|0j5!ENWQml8OZw%>61a*gR4Kzi^+WhEu+wMJ)xv&>K7 zkb7Q2P1JJNE+3$yG%6f&Pb^(z2v3z8CclQU^Fv}1#W@rEkLY|L6{s3 zRzW-ir=>O}RUk0J-i!uSdNGtSAhtc!;h@lfh72jjox<@y?jz~fN1_E4?R9-6&Byki z)@>!&z5~*R(WQg3thbRNrMoK0Detjhay05snXuJUV4G(bMn?#lS}Q{rH-q+LhgB3i zK-PgAIwJ5fFft--LkQR%fNT36Rf~Z*^Z-Z;g;3T+fQiQgwY5mig}quBjwaN? zu8M*HWib>WQjedz60gp)csjBt$2bmtV*n&zZUB-qBco7@iZwY3iTznE*H*ZR8$jh| z?{F~h9gddXkBt{Ou@7Be;BLh(j3Xq^wt_bY`P$EEwGzAzVh&l^@ z66U;a2zANw3HAfjSMxC?A%e<-;a0e>5f+o9>Y$dq1hi7wwpDf$(<=6xHwsgZS{HvI z*lXH3*YDS?o?QgndJNUipRUZL0DDS~5TL8Y#JG4{a?~bgsG_8pF()$sAIXrExLmuH z3~(@jTF?V>0mEL{p&r{gm6Mhfo7+6v ziNHu*-ufJBW9H*;(m9d0p%Sz49J*;jq}m0OC;Y??q(TBCo@RZ%R!E1*b;iV^}UB_N>)NGsh4NOyNBf|3FP(kR^^9fE>%=b^g~ zDcx^vf&2dR-n^N2XYS0sIC9S3``h2Q)^GiSDF_}s5<3UU6;9!}s3!jcQa1u8;6jmd z9JB2HPAK$-_;g?ZiWD^njR`=aEz19PTP%R}NzOZsTtM0L-N_F&u&Nu+@Y_zUf@g2B zE^)Kb$O-G2u5oyqu4>w&--KerNsNa*;y5)2!>QW_Lxc5nO7}hJL@cAicDAOz#oMi6 zTs<%OZQAWy3GQ32v`+5_oQ&t0_+JoU1IkY!2t{Z9Ry9_$iW)HTo3{!dUetdtvoPLOruZG2-|iYAPnuk-6_)iI?Fn4@ zMLNk)k8Ifalzxjp5W9N3rTT_V((w~lj<4td;CNK40hFKp_%i;+s2eH;5V@#Nf-=#`z&GZ=-bbr5J$fSI$PuU8=1#YsJrG%v=eS7* zjN|ow$Mp82LP%Od>J@oI6HRA+#Gnx7(`#k*AMtq=ZVD09-}b|Qm+Jga5;in+0cl+@(4{{2V3Ur7-ICFN3f3vrg;LRQjx z3uA_ankO}}D8hMs_Jy_=NFzby@B}~jy{aMn=-%TA)sbx1))cWOXqaL0ruz5`-HF?ZWNB64o>BrJD%LQd4VkdmX;+b$ug7nwFu0m*F+G&-tGD z4cJ-MKMwjHh<9{*UHBqo?x9eR?Nk)KcZzs==;)+ZUeObT$adF2(-qji;6G@ziusY= z{==EJ>K0w%Nq7O?`boP!W6y^ynC+(0+ zH9~ozSR66O=3G>eUKpA}+iM+dcu`mtL7}#G)7iadXxkgi!;1c`YPcOfv0RcuY1{WJ z#ixcc()N0}8cuw^aqh)kaD#y>GD9ZPj`#}Dt3o6W>I~M>G~P~u9J*XmpN+J-;8SB- zI=KFRnNgp1pFSOR& z0A4{*Ehr%BWOh1K)@Rz@fyJEfKg$uUcPzb0hmp-AE4k{IV=2M;-F$Av=}UH_@igD3m&-#bI@uo`(9_Qi#wUD>k=5&D%+j6Cr&20t)D*8 z;E_?mC=RSAx?lokWJ!a1o&3Q~<~v)(rEH%0n?QA%_W;S=V;)pM*=2;ri4HkAS zZC-XuAUuyNO)_u~N){uc)%b`F6W-F|e5Fh=P62xwE3&R1$B%O4WrS{TxvUx@$( z93e2%oJ#5jmo!0$=le`wy8Mh>8xV|^AOkMktnUwGO(W0ltd|WnQ#3YS*f+<{lt?jM4KS@?G6l1Q^&UEBuM3(}vd3-HDN6(b<6oRi9zq6FAEK=ViNrp&% zn_xQ(y@MhnNoR8ZxtU~Z_)uZy@YL5qqmB{!YL>97$smP-^zFz*A| zGg{4hZpWWIl~1WycRt6~TCb^F*Qe|R${-Q?dC^S(yia71mKPD{|8@M#%d>-4B3a8bjQKZGoi}m#tKXZ2)U$92!goFlN`(%GkftT-@M@L4(5>bj*#DbXZ9ahgBKQNQV3!wm=A;3fcl2xsL^&gj;K+z+d z(-ZcRQk0+D+{KycWo+k zH&asQ)~2|%bkNP;`IXD9OFArvJ|&8*q?gFfaWiHH^p6|ukRNv_KOfS<<_us+KwP`T z;o(nW)H5G6EdRZ7uLm@@L>&8wtr?8^vV}Q3!#B}$DI0;O@ouD3^UD zYQS}JiPlqL+sRxRO^nxH+vXHqF$}lD4BpdD%~)LKKtWlV@nrO}rAXZaAII`}RfWRQ zj@#lVCmr(pk|h&`0>D^@uK5h=tzSG=#S!1F)MbD}F4D3IwN7kAACV!SE2>(W&qL-n zorT6WFScmtVIUyy>SJQj-N>t_lba7b0oq9vpC#}Vg5nus0@$I0_M{oh#k(3H7iKd5Kq>>&pDOZPC zs;q-1b;mkar6py%CjPrD4%qe7^z`OlWnR8`u>%`3}~O zX}6G0pE_2bV{&o8c`1y&AlPUiMQxDd%zcm7_~9e%qPTwTt5=;QG()pr@BCscr{h^LUQRBtM5%4rPT9m$u1RIGI4|2S9&a0AASpfssn%UJoMZ(9{~#;$KHp~RM~PospU-pk=Mt`t`ew!OjDBRH>8VCu8E@If*TMJb|5zhA#crr z)knvV&7o9ruI=aWbD8)vC<^2NI|m2Y1E_(H^hpNI6i+@u?GSH$S$bL}IUyT*s&|gOL1U7w7H|#_Kw}THn2?OxR&8zyaEe|!+Nn3Kolb?9Cngj z#W*CxOgqBV{lWzu!E-19n)@u$8_O?v?|SMzlGvX<5)*8IWOJ9mfT!^z2# zlX^@#dIVaUy4-^}aO^L`5Wm5_JOxUG@Mwd@~kCV8vPzcq20eNJ6q~a!1mRb1&0XHa+xGDJyn0ubvVnkwCpO9+Yg`8qG2-aWyxQD$XLZn@Mi8E8!PkN$>|nW4ln-T;UP&H*e+#`1A)=q7UgnV zV;!eL=3};AbinHmbi=u#1YOMkG*I9Jvajr5@7}(f&*VcFRwT&nl>6-!noWP?_8j_v zMAG44XLG$_sfCG6{P-RG$}j>%c^7ASW!O+Cl>2X}pgW853_9{%rtmx?)Nqt{M+EiS zSeltOI%;S?vsqTEWvUhQS}xQrrCO>Rcl3l@Tts^|4e#EOJ^6(Vx?MipkW72GsR*hn z$50#;6EWz7G;=s6P0`&oG#5|zk1@iKsO-yT2}N*do;+_!s8`4XPtT97O>hpFZyqGI zkI_{2Br-I~s_T7*OJSd2R#M9@Hk^OM$z}VYKIH;toXmD@!@FTEBxgER`WbHd#4+f6 zb4y<4X6gL*hA<{XYiFZv{+PhOr!PxC>-$YvGCu*9aJrjpAnj!Hc;ID@O-x%P?QOKx z|Ki7=5&SMGbXN^PK6cY>y~V)0e3@#s;!(;kWN>egX~+D>@;J4bJq{Hi`twW|_4wL! z&NMCitu|xl6E(^&FTz;pCV(WLn_=ds*I*6nma{L8hUHNOp5mv-djwsB*ygMTjID?`{AhtT)6?loa4>!0+J$M>_w*JbJEK%uEd zjPZF?O~i-KR_*K>X>d7eZLYl!r?UE)3xyR$4;>vH4P@ote_y@WnQp1aW2S?%6WyaS zo<9AE z)5qkSn}t}VJYe)(^Y_GIJ3eugs(}5FMe?ZK2icDs>tjI;&L24}$x*P-Hb9W(C-E2R1>yqwi+hb4%TLlgn7cHts zRranAK7o`Fo=&(qwjXpJ9q$WpUV+LvAzRbjY<emolB={=Bx!-E<;{L1$?!ohY6P9r988$w3wK4sP5G9hR)g<5sP%f4$UGwGnc7B`^QOk1?9A85DT$gnolUOOQh6Juv?1Pvz zVbDE0IXY89C{^w>Wg^9pfjieimZdSW?gMoQoJ9h>wLg-me{RcAJP*8ufKKIP-b z+2luwRS_0;qbZ`Oph+I`oWvfFy6t2`@xoHrtd%nhECVTYhQ|TLj6afSfe@ATATJ(x zhzFJ}gAV$%j+0W|)HCrIYdn4dw^Zw0 z8HIdZMMs0mjav)%sU`K4EO^Xyag<)E70)i+ULW|uo$^LDUTwZSQ;Q~>FBh%seoimE zQx@%QBc3T<@)M6CxhLbJWJ{MDqc-gT(d6n$CwS(X@CCP)C#&Ab!2(l$B>2`n+wpy+ z`vCHNz>3u|xB+|;^B`pwNH4QMkMD#S`E~e%)U2$mHgqA(kk^2;BNPB}sQ^#Iwm-chU%Wdj8Pxr7^b`7e>xx4p=g1XD02_Fp6qsZW zg*!%OkZ)%j?it2P8mM{7-MTh6fbi2*H*u(jF6!_@Y5@D`NVMa2Gw4%Hdu>%zG`s<1me~ zn+nyQs~IpV$SD0IgfPP(%LJIEm`)&$#V8Ep?;fO|+Ec~WVS&hrJCLI+=u~vz8Nz^$ z^6|QqJx;E8OW^s4)d$DOA_>;A4Q}4~{0{foQQ>N+R;Fe)w{7iqRT(XpLx!O}9b9eH z^%VDfGGSLxlU|UZ`mHLQi`6F*>fOkprKhS17}f(Pcb6$&?Y~slBkt=s+o>>jZR}rn zTbZxf^DLUVfIZerNPGl)=4pz46)>GFNbtKzUkJ^Q0J-Cuf6E^ztlq=OOG%aFi3E~Y z1UFHkz$pD*lugA0t%Ck_h+(Itp%DTXdb@ES*0)CRSQ{o6NGiVljQZz`zmbt;J)4^- zdUq?IeY5&$!q#Hsq#cHwQA%1KquJ$T6|Rh&zkPSt8GRgo%w#GAbM$H#9rv2pTu@#d zfWdY^9FT@qK6se{!7A^LZgT`1R|`hR<_Vc2OwR-25H=dDssuK;ne5}725Ek~9ESa$3HLz`Fk?tlbc6ndPVXK52X<=n8>0oZN$^)8<;Dg`y{qYl54F2#oQj?q9)n;Ko?}*5BP+Vt{PhM~|I! zLLnd+06D<`u*&KiDFvH6;Q$zqrq9jI#mM8v*sd18%X+Xpjfcagb4|_CFcO-j8_Q3^ zVpol>Cyl%3KCE(%j!w)RmN)BJuX{*pu{v{Dr^b5lR#4h7Stat3SbUHVx8E^D$}D$p~2u^cv;=LZ&f3!dzP?Kd$A@SF5HnNPtqHtt{*LjBH8SJf;TXIhWkNJW!3 zmz{oydr$5{K2H)`!1RhvXRn56HfMkLxxLVZwp-2k;W>KmGDx}^V{DnUuMPI98goCA zrQ{28-#Q$UOUk11E558_TD$&3q6MAR92!4!wC`2`ABTz!*$*? zEp)KUwEqx|;KK|PUW2*>2PoxSrrVXO?iO40ZauwV&34JuPhe-PE50v3c`FP3ICitV zlhV?Sn#*`xv3`K5?!N_`r2>rZgv3OBC{n|~q#O8%X3zDf8$%_XeO}`c5J(?xyH*vy z18QI=LWd`1!r)GuaF-4jR6d}lPHsb=c|FylO~sJ+uKv$PbiaaRK&IQ4TrcV6LkU;z zON;F?j50<^&Ob^@CHT6?;uEoI7#x_+!8=gA;cD_@yY^Di;BL7STB>U{XF6miN?Tm= znNTN1#?3*y`opye)*AiksCD*-XHO4#9MR)pi$Lh{F|#(l$PX&A zX+}7(!Qz!#&<mhit@j!*6TI9^OGTA*=rGLY`74WzXLVYL3(9Wm z$P{bCi#B8tyw|UDp3CdTD!9Z_<>f`QR$bX4E$=k5dPYO3uGn%OZT6n1zoVEl&(5(A zi{^gQitb5?p9b*9D4gnai5k_Mrmfm*lqH-x`vlh*t77GFTV}R^JDoP_@Ok#JT$Juh@gRtz@LrhcBsMr7d zf~TO)u0R5|Awc}|9mLbE80130tI0S!eT{%X7Y+o3CUFP(b18UXPy5fQ0Rc4y41ojU zAnAeSu`xK42gE^sVuR7}p{&u-_w7~sl1hcWN1x?J6+%7bFQufCtQxF2>p9KcR_AQ5 z+}nSl(aZFAbD5UQ4i4*Kf2K;V^r5}usG8!7O+)AWW}>w()Mc7zGtn?dp8+z@qN5^( zhbkvdVQoIAg$4v|u)iN3Y?dPt36ZANM0ODHhWMY(glb2%fo3cTWFyKT7wiJs^F1K6 zdwXBv+?~&ni5rN4VXbPw8p2V6>q%Gvw|#v$>I33H-}hjq`?SuG2(;tUglF^fv^Vlk zDepCUctjQ$>AN;N83c4sUpsplp#vrK!{j3`Dp1`c01?hKx}R83xs3@=M@@)$>(a2P(Ip>!e?+NK(tm2 zBEr9iSCgV^R?1BiI)2Q6k@D&vXK5*gwxsR%@sq_n@hv*6bEik?6OH!z>x2xh7~Tlq zYgVT_Txfl-4oz>Wn8V+pM|GLbYOyie{| zeB~e`PQ@9N#3^!;m5(Beej-On(0_TwMEmn1_bO|i=;HKh1TifhNIxQ0{%;)!MbDYy zDRmI3@|0LF$*7GEb(?`rx6q3h*tL#|d3x~+{RQM!Lr^BC0UzCtbQfI6k!S?kdo?!s zAj0|q!i49D(88lMzYF6BUL`TQr`APl0}`8GR&A`WXD*ETvmazruD4`yE3fR`+tHNo zzLLoiuUgDix+?)!W1vvLq3dU08t?=d(^OOL^Y7(=3cT)59`OVwXm<*QR zf#3Qu4C91U^xxmNm>XRx^FN1_OMmymzuKXb`f2J5J)XiOyd>$yns!v+#*YS}#0`>7 zn?LH-fJ~@U$b4F6zH_u zfjH!hS;FL4^M_e9`6G6#rbEpiTuy3H)I%wbrHc-8xPL*#og(=_CfyC@1RHelz1hW% zgxcB1Yj(~f#J2xi+Jt;x_Nd#)5AY05DVv*~ouGeeG%Wu?R1-)m;Ocsr?8)Oh&`?Av z)d54Y{_TJZKK7=R6lqs+dwr~;hdZgW&^3aIr8b7>45r_!=3eWI<2jbAw__(vSR9yE zver`b;&qA=(j2<+T86k3qnmdF>B+es-Yk(gfrXSSxzC?L;GT(%1R!!7@hp-O6PZe3 z)RhDZa6GJb2=#n4U zdMJ?-_{S)Uho;po+wcx7BzJ^&Ftj%ZTrLwiF2r2O|NcJuTiHHu7@dl6J8>%79E+0_ zF`sWMbfjB)@~_X1C3@c!e7HhCxVB8iR|S|Jmz_}tOIjaPHR)p|n4LiG;Dwk&Sxsp^ zJHz_N(%tj_`}A}00{xf^L@>OiVz4zSIAj*)C?G4YMVAcDvO6zJCTw*N(yli*(tt^*jT+DfLycD*Afl-AhXCSdShy5~2r@J(Y z_j-)7Tl>(h(MIQ?G{40%sl+v5St*+@y3OF3LxrR88tCcl3!Y^OTg`O#YcM#KQj|oG zD8L3w>t3D#vVDL>_op_{)U1~;wccZB4y+m1H8K3s z5wBS7ki3=DoscbsIvC2{1SdgqpjQl!5^%DL8}Gmry2arHeQ34;VbXYhP7Z z!CWkUjW?~dm|rqPTmExfb`!M!0ov|N4#r`(8A{ zF2l27^}i?Ml8_B_NRttW5dos^eVA8;F@UMT=^pjYa8q8pP<2%Xc(v|+x}*Jt7rnoJ zCl4IhZ|u&F6V$`F=M#n_0jEv+UYVii5=}z~rlVO_l(ZcCn5A>(BHS$20h4!kX8i@o z{K4YjF)`|RH{dC_&e=o06Hi+)``~xN{f305r1!vxwdkn4O>mW5?Z)m}{C2%U+5RUC zW3f_(Ksri&)DV%K4H~EisV-+Pb?_W4syM+O075 zS`?9VFzPM|1J9KIp2o-w7%g~z{MZKaz&nCSPB-YS31AKj9=iy#vh@$@@gu}?fWDXB z+9%JD{6`*9Wa;ZXtk4mjNV|l&dV7!XF#^plWF|>NXS8+A_IPlUCp4pbWB19bN_yX) z9C7pGYcrXEhaVgil$@3Zcr^nJn;5Ihoca{(43vavG zdiWcM9TRqtV58=gJhU?{A}!m+Qpob?i5Se2(~_2-^A3!gV$x?rbnI8yRTEDJf7qZf z8UaOIE;|i=iS&=tx{F@6YTG7&MMEMTmnxoCF3_P5T!NR&HFo<*CL_*@@ggK>ni z4F->3fvT&Ia@##SKF2(0z$_UGQ=WGi`+TC*i;PiS+N(>CeivNJ?CNKvM*}^l!N{5^ z>lg=jDVG~k$4B8^KNk%~Ye87ybhW={Iu8bQ7;pS05i7~qZUFze*Fy`e>#NcUgYVA~ zcVWcYzxJ;uY%SRiV^aBWcm6vle6C4hJuB!>2f_LrxZZamY2OG+G4~l2(`!NsrfM7c z*YouxLkDfIIGrm_EGlVP}0xowV2J4?ToKFpO76b`y_t5 zmwb!q1O@lF#)MTN&b=>%ocJIGwOGhQBg|(cp8NHe4uFVY$>{(DNvL@q^_3Qpb$+1Y zy?^}zY*12jC;ACl#QEdL1O#AYf%huiw@CD{uTy)aB+@y-!qDr&h#2o+i2SQ4B5OX1|A<% zAq)NYcq7tXpg8)w+ZZYb-4$b#Eoms&xvhEh@hBB|(IU7Uix0p|pdL20jFLqOz+K*P zoDm24qF_7J-q&mK|B2~+T*xy6y34Lb_kP~9LdaTOpH=?={6C#Jvp(;$;at4^vk}B+ zL&r+2f-z0bjW4A&4d3ba-@|QXzI``ztcsw24LDWO%Kav;#}s0hCHQOyz#=IHAvFvQ zElB2HpuuXU^=ZHwz>4Q?F{5;>03jvxJ6c2r1D{k9dfvdt#Rt|{qCaX|!4O89V0PcR z#lNI^aHJ0mXxfSP54Aj}@$a{Km>UJujIZcs9A9c9yyVSD1Qen_Cb2Sto{~}!tnHWe z#ZawEMG}4C>c7lzC=B^|Jpj$NWxXtsNs$hN{g#G~iNpJB6FWI)pQJf0v@lm8Qm*p7 z17p?r<5ni-AJ`nhG5(Gqv{%f(4bPFA+^HKi`J#?;spcrGO2U7FRUrp{k@#Ton7YLM z%8>nDShbcz9)G!0LB_8bbo5-yWav!6 zQoNyrx!~1?AYc~Hzf&yw3mCT)iu7BpF5w*Ao4?WyFqIJufH5R3?*J&NpevUMumGrY zmy-ddrHRIV-w_|&1v3=%5!`ohVB$$88C_8-KJ72B3 zn4FT-6^`4|y7>0bS+tcnak8*`nU3cYSXwG}FuaO{5ytj00$IZjCnW+pB7y_J_Wr`% zU`3wtTl%7nXHna(mjfv!p zOQhxxZyQ1DzaUmNX?QJJ3O-rJ2O91)4a7rOc4jS#%;2$3+H2@l+InY11$A_rU6!g;xB5`;J(8s417Rn(G{2+In%iE_y#hAjoV2+)7 z;ay_WjPB{;4lSnVkv9CSK2Hif02-*I|FPK#5QjNy;l|jw>Px%SK+zRl+Guoy<`5YFe_bM3G`lL{z5v!v9CJi1- zrpjcS%$e zk1H#ZoVVYARSn8ipbV7cf*t{2WnBmxRmfI3R9 z-?Tr-1hXm_pIs%PMwm8Bq#591laQ@i-q7BdiH6OM<&iXS1{JO}gX3AVax6MAD)U?W z$d2>zC%dJ@Py_D~GE5;$TM90-_aQclYYJqPUo@WHz08s4%=(^uK+8g&5Z?Lcg$>EX z2TtE%8xOB8bF$A4ZZRxuMYAMlW)Q8EQI{o}dE6D_lr03@g18uL^Soe>H1_e8`30G3NPL@cC>;iAH@ z1ilE5uFxTlj!IxZCQWE0qdO!dO*kp9;zq<87lN$I)_`t`r)VylfO(wp-U?=EOS@Gl zhx%Uijr1|@>_?xqDy_v=EnJb@FWfVYo<4b8!oYpLiNwnl|EZ^C1|4*VoQg^mA%jX9 z>yQnv^HwGK8dQI&Ai*Mha~{>?@t>5Dnyxcnekmni-ESaI`a8#Me6o;lpAWR_i3p4Z zeRuo3WzgMSH5++EK9_JgehZxmT-@)lSXm(2nW)-!--;RxvKt>ID#pxmQ$1oI$VKLB zbAbo3k(Ba%O1Yg6HrBtD)6k7R&hz7}i%veSXqtj2Ij}e9dt+fF*M05W?m8wB23oR+ z$v_k&sUp+1Ido?9U;CJD;Ot5(^ey|IM*SQul4zX9TWpTX9MpG~HnmC+VxfDry>XdF zZYNTc<26-q+;tyFmgX|mnRmF|dR}QtO zg6bLM7gmpXRxTQ@@SUP<#?GfJm;8)vVqO>3;ocV0I z@kwPzS2vh~#rBt8gGv!9R}MS_BXpkl>a1-13n;v2~h9D zOtl?$m~;EeM3y9Qsen((=CtzHegUyp0Reka$3mCZ2s!X(hZ-yJ4w7wgneZItER6lS z5z4UKX_QWBSo^7xxAr zoVb4o<F*1b$u*$*wRnf%J@U# z_}Mo7tI6?nZ)6!x?5U~6!74Mb2qCpvECa zjL)Kq73xMmGd>b~r8|4U>)|!aQxx>|@%}Y!TDe8o2oeBn7@z@(Xbg)f==CYv#30Jq zS3ee+s#x*&Qd*{l$2*Q&T$d>*mDTG1Pp%BzOhNwb zV8i`#uZPr(zq*cWDq!#my{s7rr3J^+$QI|?fQ@gv-VHOFo}wy0d(FSnAhaMdviPJg zKnwC{IY{tHwPq>Y0RB_wZ9{KrF7tKTu)HbD=&9p^f=L?+3Rutg1Oa+#I}r?y%zU0o z58!-Kf87FIH+3mwBAD~O(r7ld(5$mN+I;j>-QSWhLsC;TosZ~ivO>O32N0nsz&_pZ z1vuLo@0%xF$X{3~ZU(BY`>SXg=6o;R`M@G;D)4(%{d3$5g%Km2nxNCUKiyin zx!)B!WLveEK3gtyNQ4YBcOEf;{2E{OM$u#fZge~)sJby`B%ZDkZRL7l!{Cp=G2x6S z2f-B)T$iq3;2U^z=po|Sk$1Scj=LK>D}XFw5ErzVr|+163yh@E6i);Q-*3E*3Oi?S z6UU~7D=8-uaBEC5hiya?WlT0w_ad!q$t4{GR6Y*bS_JFq z&y{BAN7_UHyN|c`&EBEBf_w}*AtJUuOrrdRxXE-49*Ty#IYcCn+N0jmAasjI&iJCCG`&`&W!B(xiwou*Kpyvlzrgu*8&r zA5=kFv<_q6{RlgG^1p6*s)xHcj(^5rL`9Z~*#YT*^US()!d9?y!}?+xcm0Ct?vvxz zC8j8=lt&KmWH9THECsq)UfvY>-eUOD!f8~i&7y+&MFAjY0l_8IN=u%O3tomFKGV(c zKGDaXW;83pb z^3*%pbrjQ|C+_=~|D7ewTt0MWte%HY9(IyTozW*vXw>y7-cW?HHu=G-2Us6$5N&eYE z`ez^(=crcd5#~K?t->EJEaoBSWR*24*i;|heiz+oYAD87<*dA!pq1g&HEnxbeSiG` z@F&S_-#42hT)zsvKC#5soCOrn%c$XH!9lu+hOpN3j0bM8t+I9jhsA!aU)K@jyKSut zasaND*5+>+5oC)-kT|{ zOw6}i|5{+E0||(Xl%`Lp{^vQwjv!4F14L$4;}YA;Sty>haLe1zY<6OtomkzTCWT33 z@~qdRx7>XE9+(6V0U9uTzp=YG7}U3H?~1lW&|Y`{=z1+ir&LK9={VU$Z{TQu{BJY; zYuWqw?0j#oTt-~JCnn%)AwAIr@fU%NG*3s*>zWmgiL8=l6z&wkpZ4YGW@E{-9yPh> zP-LTW{nUAjpY^~ouHqJndtpg)>|t5%^L{fcdu3~-eo4!N5n%L#>TMPn{h#~CH~!l^ zTg??B!7}D>tOQ5e0sVD7(>gOe6%OzBRdG7g6%2XiZD{xHHbeX5&Gf)iky^2lEHUAK zRgen|%k%n-H?CAGDw=nKm^7n!^9;hv`D3$DlS-qvUVWR@y4j7KUM-lp+#cyxgYIt(ZVNHq~oiL|mVmq$AkgPx)cYN5nv~WW16Le7f05cLPq{mY}j{YLe0*{utdaE zc;MYNnkeD~S?6qmjgib(E}JRVB21$pZ)F7i;G-$wDuFx}pC6kT%X(_LiI9_;?zJYI zMK5e{=Bdx3mWj0Ewn-}5n`rOX#;rg(%h~EDypd1Dw+fY(#gOQ`F8!>co&twOEyvYG ziA>B`Y`X>KOD6u53DFp{Wf05**bu{k{~p!L3(2QOeMvd)2tryqq4p`%*M@OG98GeB zR%KGYQ;9>k-LlNhMtP&BRz4wkq{&x)fX{$-yH;+m`Q!PW@%?)#n!9oxJ!=fI81xd_ zRvG=!@Y#H>~`=8W2aXTr3xH1?85| z_eb3@dNty@i5t;frhOjR>XXBfH7ps>fbg^4YEt|V09eStYr_GZDl;>{9WpBH2dH9s0On^f+#4@^E) zz0c}s6#5xJ{)?Nl)fLLL0D`l^g|Tk zw;Lan=Q|XzIIMg!E3@Y>K{w@5VR&PksM^`9F>~C}RN;eKA7rIF=X?Yb^sS#3>H2_` zo8>*{B$@M5q)E}*L#fh=P8*8H+y|ZqCS*>Y#`{p$xT6te33|&F2;XC4W1X=AxXt8e zTwb0zEm8G@hB*RLSMpx4DgSwl(rQsS6 ze2NMyzSWNx4*2&$E4PigyD=$ioKJJP?k9LOt#}OdX|3UhU%qlgKj8yW(kYuaHW5a| z9P;8K%`a`&41fAPtNCE#7N$_&+1dXqz&B3)deE|Df!d_|17RavaYIwjp4 zUE`$mT}GNY&lCINRzJyovv&DG>8x}q#&Q*n55~P&!humc(yV*Kb>GQ5ee?O0WP!$8 zZnu&sIr)F%*<3eI@NCyNwq9-E7y(2e#mKWLCVOOKnTyE@vKi3fyAxNp=>78fv#@{t zh|@ctYNul2#lLj-%!^q*UR8lMO6zI0PvG^d@H(**sZ$b2hvn(T@)7-_nO0)u`QZT~ zby>Qe6i|vytcx^}rPs`S8;@qJLhx=yM*6^hNH{ny7oJ5<0wz6UGF>DR-B3rhNIy&lIy1?xG=fmGb1ew8= z^palQt3mCDm-1>KtI~NW-_`CMV)4vg*d>~>pVO1dxAZ9x#tP~mrgc7R4O2~;P4@^4pR|! zFg{1lGF|%!B|*c>s*)P)Ys^gyP0D1oYrYo@`AuVvU=OPQV2@cgupVYgH)XG|?3iKr ze0>lY>c1k0^Gs_T$|sL!BWJ??-pAz+y3^@Ag^h1Td?UbXD)3sX)|wm;LIK7w37EGU zXCQz{%WS%HV=J5w3Cy4t1=eyV!PEz)-};yoCeu0FKI#0bOt#~v<2shh?h4a-ax<_Y zeFnMe-L#pd-vFbXWpLj1660L&L}{Kw)dbJv|9FFN-j|Gw3`Ew4LGvk{#C{^^Nb&ur{%xb~*&B7IEt);omkDqJ6u}wT#}o0>FxqoOpuYC&Sw%abKduLUWXY=AqO4!Ei-2CGe}ps z7Mk1cpTX?n;nDmb7OH2$dC*GF5~_3nXO03a1K4Z`6Q=-d({=LI`!^Y@gv@2-KGpdX zL)i~d)2oQ>nu9rKb$dyNg#ZV~gT7Ol!xoXuBWmmV&KcBEMF+h5rDM9zamU^58If8r znbPe{dKC~C?R?NXCW4CD_M9C_=ed4k8ZU|~t0V$vT(RFzxI4KxzxkUt-X-v$pfA_{ z!oKn1p-ZPK03CZXkN@Td?l{XN3ZA z($FbW?+d+n%ifH4oTMV|3sq!iP9TL0aIoZQS>--odCOgn>a?RVa#Ql_0cF5GP~> zW1!po{S%vUr+#?bdwjE-RliFBCcjNUaMkF>?sD&NuCAynb=1H8f~-CVN>@Lv@$1{mud(3$M$q?ZYJ*N8Z94^;{jk0p&pFe-3US@3;>_mzsk~u8@;)y$ps!8kFaFi}J z@}(?B!N4S(7_RO7Hj%8TO1b{+#5z^_c4FJblZ|wKS zdB|wv_L^;N!KZCfPUv*imjBe`&v7p{=ST!xdYkn$mL&!VJxSoe6d-~xZ}Wq`oBCJJE&2hS7nRE04B z401@!xKJ^(>V50XRT~rZ1vdj@&3Aq-hCP_&=qlQb4&SOA%S$OM78?~C639a9hHSo4 zJ0PQrD67M3QOQ=0E{jNRSR$hdH*_vh@Zc1IC!PsRtLN|ItC`97t-KVftz}``J#*9i zXi>%J0_-LB<-xBxd=%;&cp%9)LN@K?= z4{rEX=%R#|NK%wJ8uAV{N!_!tBw4BArM<;|r1?>hjSIEGkQS3{?2(+3pT?y!ozaGo zY(bv}RuuK!Eyb0i?rfw-tAPC<6px9guCW2wtftHKb~6)dnom?;o<)$XtopYic#VkC zXcK)!$ndKEjos18Nd<64qaM1O><2I7X-Nn1Nt_=n>q^q-;N6qb>!&Q7Sg2A@!zJL5 z6R2pNO+*f)N+ZVKPIBL6y7DYxu<)9Il=y2n*^PM60(bP(Ff=kQ(0`6QAZ_j{bUg%! z?izg?2)%(I* zDDF2YYdyUyaF2~e=-$Tsn-2^0{ry__kLC0*a>|F!w6ED&LsU3AA5Cml^KL8K17WCz+-nu@bS{pCy-#TyE0aVu;!WU z)^sAlLbxxxa!(NwB!<0=`h{6aR+9QSDWpKaWv;{MdAqep_M)=W<^#!62l2ci8yj~a zFWSqd2=7pQ$FGkPl8!@5w+?feruCiUj{$n7k4+k;xM?ubK&&nlO016D$!~C2rS;_- z(h5SrK?kIu=Nj}5kqZGhE*Tmc>iO|w_Wf!1&hBnKP*EZ>Ll6)u07DPhY`qIUnUzrE z@e#8ceXbmR8A7iN_E%oF>%S&r{M?%I3oOYEJ6&QWrKIi%3JD2)e;nveXTRtWG?~uy z`_VxtwYF3#9}#J21?S|P<7Q5Z`O+bK5Z*$2{d5g&<80Og#;v#IQD0TLc9n;JXw$n} z&jqFQA29Q)jaDCW@WmI|IsAFOkPP8_{cmJGc1Dv@=ETY6Kw?$|dK+60V(@+Y7xpH4lP6 ze~v>^#GQBhhav48J;IDw5JH%Q!0L2MreE)Z6{KPdAqolD3v6pi!ivdIr5|;N(NZ_> z50*YVKPa9{6?;1;vuUO=hu^ z%_#*#l&yLZ&^j=Deb6P(f%3yeU|?{`NKgU&<)jbUkZ9c4o0pBy=1QguV!-Yn#OXK!p> z_AB z{f+!geUXceh&SQ+v%8)_bc*=~CSu@pdC44i4W2!49GcmtlXyLF5>_a9-F3E>M+4$u zzTFAY6C*DTtV-Z0d-xkBHpwYeE#JI-o7sqdh&=4`{`L^qvoqD@k!e}j9u$iit22=` zTGU{r&6WR%Q5=gq^45{tlBoZ&Q_|B0;1t2|oJuh4f!bTxdWa)rfyE8X?H2(?FfDMI z=)V3p!9lnFGj)9n>(U)WDF?DK20?b9o?q5L%x=P%^&!|vz|UxzOrzS{|4vgR2RchX z5$xY%9;$QL=rZPq_D7=tOAvhiR4R;k2WVJwrNJz*I-h~LY|-`OUai(11Hx!VwE7FZ z(qE6Ai3046DcYt?CGH&EJf2oCBedC`48a5|8nuDHC^PV89f2bcIR^IoTOfd2|8s9IE|xCZZ~4i|h2gx8&UIAl^@j^hCHa3bmB85P^ro4N z?-!a>e$)jx;Mwiq7$=&dahR&S@2(F*jUNc-{zD%kj95e?OLGuZV0JTKJuZzSp#%4# z9lmkzjp8QcVPRo`>wXH~tg8n6DrDL$-c9biFp~c7WeA$k>wJRZhu{X8#8#@j>;=*6 zLCm%H13BCYvgeu6ud0S5Ifp;@dh|>ZYlmpg-t6L}pu$+j%zWh>b1t^e_Hc_=pSa~Ob zq?g_5-=k9KW-T%b{ma#Rf7cd<@tBt94r^@!%xV>9-g|g}wZ6AK+|D&bUV*skAU5G3 zjO;?pHBt`;6dzhdhlTBPyA4|#s-|&AR7T1c8hfZoNJzwHp8t6tKkDn2?`f&Rd5;}* zRIwhP)Z$B#5-vh5%Qpc5aaZjunC5TYr_XG!-}@KNlB$ZvRw-Af|3%kVM@6}RUk?T# z(x7w=3Q9$l$T{3Yn} zdFB%*_St8j0K5H~F3d z0RA337Yy3@+O=qYIJvi+;-9;c90Y?mX*Cyb-M($(F#4!slTf$2%-I6id+I8RhAcmj+HO>se-gTH&DNu-2 zPPyW>>vcN|Qk@GbTuQFC*CTWOw6)VJcAmpMi-%WeRF7?DVZrc|nX(YLHHW`|!U&B) zpPpabNKAY8uGi15WXKg}THSgsd9?3bHu8f(_~px&^Pss9c_H3sb1)$P_ZvC+%d?4Q zS1_sPJEvH+26Msz4y73eHQ2Z^OfU{};xpGryPaR_(hfKOaAdQoUAm_h!c^`3O9;aU zRZ|{uSaA4fHIg!bhg0ga=QiTh$_WvJGUWEqy!K{&ef^s6DU^PdX9lpLp1<`wSTMj$ zcDb*puKqH?@9@D5j7Is3xHuci8kie!@dm_+JV)pzk4!y@!%=9nGy+uAu=PtF7U7Y@d>2o)}f)s2T z?97fg*a>x;J>S%UgyBwWzSq{0$JFHwj-!$p(F;w=5KA=zzHjN=u=%d@^M1 z`)Wa5OShe!&Qi?9tjMRQyEsQK|EZsQnEZk+R(7(|>6bc~0F7N-T-R^6JMp(ih82d@ z3e@d~;0Iym;ei<*1yWgR$`c1|Nzmn41?z~>%&ZF2P*a!H`Yg5fSF*hU0&!3;i*p$bo&tl395M_eC z_Du*ydQVg6CJhV>fP}{KaM>;pO9~yQ7HT~W~ws?okYRunb z*v{en&T5I5+nB><{I4KiO^mfmcVls^MqJZy4A-+*)#gBc?7MbAi@<&HZL zfPl+@L609RxxTd^ZQr}B1A_(#!sG**=kEG#EBkzl(HT+idolYY@h}M{pl^NnqlqcZ z3o&{iJ{9uD?!ngBKmRkJ#ji1OsaE)n?s$>(z(t%QxdY!d>fW9{I4d$;5#(pGvgY~c zJ`HB(ytj2wEakgb60(!XTdr$Fo%LJbs=cd@AW`tL#XpwjBgBM>hwH32zNS2rv~pxY zJgg5|yT`~u&~?JdO1DsPwyTzyeb;Ys#9dRhzYlxW9nWXBDvO-vl4ibpKC9L3H#{3G zLQAQcb7gaCcZ_cvKaYJSYhq0Oav+b!OJ0eC@#_mVI_!YUV+_7;rXD`DFmK1NteIIW zE8iS+KCeOHT=%-lJ&M$;!#wB(iB?wXYyW#06N7ONw5ByTYEp!JBw~87iSYFTjPw%u zDZA%;wtZ)A?>qmXF}gtp97W7Wxq`(Lnd3b?&&SZQkvJHVfyzc+q@6k z2&S(@*5stAXW4r$h4zz?MC@FW@uY6Zbb5ZhMON6Jf)#)9ea)BI=}hWHiO9=}GNv2l zTW9P1YHH}nA}A$`Z54MY zrB)7v3bhmBZyZ0Jlhyu^bV{qpDoVs&URIXHVG2(Mk=}MQVCk_;J9C(stq8`xLSxmq2JkcXKaKKi!nBreK}CmB>{9eN;Y+xI&>?G%HY!9oAGZ zRb1QEH)6=KF=@_mlNL2!ey_HS^q&JvqCC`u%|t>{^cvaY0HVB$IX20Xsi4&J^>MpsI?iY!S1Kg0vDRi&D)9 zXx_a1dDgKJ+&^_gLA!p_uGtMv-}tLrm)hb>arD~*EX?#d?$kcsKq>1!kXFb%J`p_u zL8FX?`T6&Ixyp12iHL~ssrhm3t2WO6{IzrD0TSu#Z1+}LGsdh6?53p?QA;<(RSlB1!k})w3n7 zk{;03&OQ%$(?E$m!>ISKJ1z@-1dvseh;bs{xHZ^TF4kO>wAM5deEB{ zt*oq6($;9>DeAbneoWkj>L=FOYIp+_~d9^3G2p%W93omxufN{Me)_PQqEaw?k2(iV5kld^#=yHpyNzvzo)CT@(97mtJY zDP!zF%7&7CH`1Bs+V1TiLfhb&83a8dA)*S&C}Q(2@DbfiJ8)_J&!uAWM6E)-3JcND z+$`j^1fxq42mmmw(4ll<=MmvCG57bsqtWR%p2|`Ct7);ywlsLKY%V{haif|ROfkJj z8g$BCqL(3|8n3EU5r@Mdk0R>bXZ5Ot+icBMsG9oObP;S5(fIRHOTLI9MaTZr$bUX> zHWk;gjg*y15La0u1-zDet)B)5(-@DAjv^3IH_aX|!XqyiUy;mn{na0=Hk0sJU4`qO zT-{DN8#g1bDwn$MCtJBsg|u-taN$u*kqc5U)n4`*@6`EpTDfL>N(9M1p7WWRDjr=C z5ItSuo72E&rVEWUG-BlRB>A#&Je;u{QQ+`Q*(}z(h~EHXH)Hl z%~i8EAKg$7F$NYeJsGK4v&;VmRu@EqM zir46+tZWbEb%nbq5Rpo})?zDngQGsxMNajI5w#tV80mOuI+0yv-ru~|nRhPy=67rp zv3nF#X@1Lra@JK@zQ<3T?L&_;dPCEfp@l^t4BBL3W5WXhNot5~!{En>#leDsY9BEe zo+coB{W=pTCjs0H5BT@gKXTzxBDi6`MSk@uFcYmJJ$cfa_ooT!Qj4iZI$+rwE|8uv zm!!cFvX_TF&#M2h9kDBpo5*k{$TD9A!AA5WU-$%kR zheOT8#Dvw}-hS!wzp^q#=2e+oW3Lo6bME;o3{rN0J0&o-einD0iOkAAE^f(TtWrAs=0 zO9S=E=#I^jUGciCHz?N0x$Q4`C5}9lx7ziC8)KWVWn`AV>u33ME|Ck&o+x4j5Q87W zU=8tu^=@2TTy9DteEbGT2T-7fVFX%eo+dDN5br#PCPGL^h|lbntgHr5)F);sk||Wq zn_ta-Pkxzs>Qzr#R;^&lLKz&J8)uHt*6j939RH z^LC7Sl5sWgop)wkepI27SF5SMM#7;fb?RMjS;_wHu~jmq<4BeEkT5VXkd>Ebh9gv6 zU2U9pO;%PY8F$WzIP1etXIw19n`3s^JmA%TAfs+_d$nIJEnP^YN$;Iq?UwgaQc-hn z&0Sic4thYgRo7osP^;;?L)B&=gnN4W{(qme!|_2{K~}b}UZlr?z2UQ<1EG-$M23Z6 z?b_-GztC^B)0nQV*QjWK`{C;hS9Q z23jHIUvDAJOvP1p`9FM_olqs=P2S`Euyp0Tk9FPgCMVY@n>H|w8n-7);2TE`VtmiG-NMkFTxne0LGu>`GQ1@@RVTUU!|Vn&(+&kon%&HpKBO^B zHU*sZ^vRHYT7lz@#VST*BfOR~`if9=|HBGTPfwJwZb$T~k-%e<)Lpc zEv+16`@7%>RXl4@V!NC+F5Vkm(Gd>Y*JoWsYqyS<9=WD@2TTi0M8-l406C^5hM1%) zl$>cd3%6*aDE-qdvdc76W#wOHH=eR(5~!aejX~YM(&+OGyI$ESiy}9-dpzygR#J|- z?TfvTFi-j|!b5t}pox^pA1|J*{TlVVmeor+t^fCo`l|ObzPOqDHa0Pl%ZUZP4umT8 zwsKjRw5MS^b$!W-SjTI-5I2SxYgtn9)y6Oa<>BilLRu|iT)2n-)EoDRehugQJsnhj z)l7B0yO_lreHDwsz2kk0*wIu&IY9dv0%2ic5y8iVd_!~jcV6}3Ta$=?q^G;UfVs;^ zL^o!&$a+KediKR-X#lsd$4GZ@m6bfK%ejgY={K;@p^9O_G2UooL`K(YkspIA1 zCCa)_ygXLJ0C?C11k8x6zxxHUw^DNH5v+{cTY%d`^=m1Q?hy90JEd{CsWi%W(pl|!7!)iz3jii!e^=`H4xh>1fA?E29z zIj)>cwW(&Jtt5T3O1=|fyX(zyK}R2ZYsU;ZQXHft2f1md1%^)~{5ftd+TfSPJ=bUN z_<@H@E-j^mQE=q`piSg_nu#Rh39oRVQra~&x2M9ND5)?{`aZo9QjsukPh8`YOm1B* zW!2+#x?J99xj|ypY1x$9EuWw2tRTL$qgkr*%HPy8l?iV7&aas<7hSffy1_vzfan^b z3!E|Xlbxpo2}zJV$=8#oaeN`Pn!Zra`E^%nU}9!D#CM`r_|?m9V)o4Rrdnqa%7Kfj z4Q=UF!E~fgF&W{|t5k}y0!f02E?+bi?bn93zF%&0{$SS)!Gh_sNn{AAt44(FEZ|JU zY;)o|te6*i9p`QlYmoWGffClhWnz+uh<`Hh4~ppUp!>JyeqU&WHDMZffHxHf6Z$RU zdj7X!%skhk5*e;4m5NpF$A!9aX~_Gv#pIneut`9ly^gcq^x67~(L?}~ht(pB%j&ZA ze;(!-RNZ(9GDW(uYNPOPBJ+)G1(C_!bF!aXN?D-6n|KNW*E2Rt-%F1gYx9cwj}|p- z??}A#DWsV9oQjvnMJT$_dzcTOcOjrO;a1nvczjnVtYNT{POgge`1n&jYZSn>wzA5D zbmzTZ#d#R1UVC`3r>vsF?4}JS>jY^A{?l>GDor0V5>>MY;1qNZm#@q+ki4g;DF1RP zV!#`v$1`=NUd?OpesunF+J%&{j#0TstWEo&1BBfKP4hi!nw1a4pMU71Jg7cCu@~YR z`EgZLR2-qz`(tnKxsy1A4YnmC0YIdpgFpLcWwFD2cfz-+sZZlsNJs?gMM?vu-}6@3 z!YIU)qQIR)*_X*HEsC>3<&o0fGxL1H54ihhw+bQ#LDQF=hkKF<#Z#0@IA)g^cJh8} zh_Au%O(bnXQv1V9JRuOqKtZkLtX0uBL)~okDM;O95+!|Bp%fn+j!Dg5APXa$aSI~B z#PCy(n6J-LL0E9!WY@%Jg6z97@zHd_!6nX2!QBJXdm_tp`*d>hUw8nn;@@`MIv8-X zis)K_Q0kvg2_`>KS3i$I!7oMK+}_?E2jHy^G?y@$BFx9cWMm9}`#j#>-bt!#j?j_F z46~RQ3xm+1?%;st&M*Qtdq<|RV!ai81929DS3o9AAB z89N)=wEe~A|CccgwLN%y{62l~J`(`{DygWTpy8hR;>FV-9C89~G8V*urDbKwfZt=V zH7JL5LP4gdr^m4N@#)`IsZA3=Lkj&K#ah`!g2y3 zd9bbm%ftU5{@F7k$(`?HRU)#orZK;>mwRtmr0%@639)xFsK;i)*b_1ehJblLssnD|tE3rjm`=8KYVVn%v%AX}WtPjtDItHQi$Ua{}J{fZN;lsqDkWA)r~j zuG*Xz(3pXHZ01A=k&~KJtTM9Txx$6g1*wCH>Dsd-hD9 zfF`D@cA)lG>)Mx1F{Rubfx&~nsmc%dCZA0;+M6?S?GGcrYvh)Rf_LWV4rRy1N`mMf zH|CY1$xuj+gtxZ;ZS-sN6-(5f0quzx-J^)y%=hP|xCYKaijP#)7RdL~C z2>u1WGU~do^A~Ez;|>xFa0(BfEIG)EmKNtML+JQQnSJ$=-zI4MHRb+saCFFUQTI^M zyZAnNrd9Br*C+oP2tZ#j{Tqmn03tl=PK(ul;5grNxh;lI-W~=TYq7+{lhGa^*Ns0v zc1gF+0@Jj8LR_~~NL+zST}l1PwK?#c939~|BG6%HdC~T}0;lILpJu6YM{97k#gx)4 zE7oG0Xf?{`86FV7P@j3tk8^xkzp_F;5130}S{f@*9Xqo$xWu9D6&MNvQ0WhVDw8qH znD$uAZ|e+Spi&)A4NV5pGQ~wn&R)7-B2k78l3xwbxcm~e^s zw6bEj@HDN-PR3R?;V=C3uYDz_^K8YvyA9HW?JPNs*zJv&((`Z{7mtAMto}*NejUy+kKl&%+ zQiaSV)j#!lhUD2{F8D6&2^xURy==b)+7nzIx@*IUJL5!}9hxwkRp)+v(e(?gWZ$!l%;y-qPDgw>`%%Uba zJUsLOJw-5|*!kV?;>;hMF)qRPS2Eky1sCcf1*#aiq~CbKj#(7uJ0w7kw^)WsawzBz zw(ik}lSkjv*AvSb4Rwd9(b}We<)+SyGuJIBu~GIPo&)2#I1!9Xc+ z_Nbs9D+|Je+@Q+g;o)%=8Nn~I0$fslgXfSIw8D*7e*U3sqC-216s(K`-Yn8sy-Dbu zy#}pFViv{T-s=5^>jCH&B+9L&p2Fj!A}7~2{7R;FuX{E=P}{a$lOEoxONy0@r0Py; z2^_av2!MU_*UtJ%GvvJNm#!Q&k!&P&?Mi{bEy?4HVH+jO`qTGL$?i#3JHCt6S$^*O z6^!Kq5Dy~|0E$QO@njD#jUm0N4*MP%@r^#RACAYIO$x4PX1C=TPaY{Diwm6I*!MCX zyk=jzvR99~iOu))Vw;Sdm|B}{sWG&eGVP1| zhU_Db;C|19G}62#TbCb(_BSGkc*o=3aee)qKk#fx#Q>z_WrB3c8JE34v9Ju=C@GEI zGw}4}>`pL2e%4Tk(>W$Bl;hU=)!k1rx(b|MgIT@3P9^J!lXTXO*p#^t(1uzx+tyEh znNm(+>!rC>^$LAG$YkyND_4gPdSxe=P;H_ zb=DcP^CXaY2nB$R@K0|`?ZE~6d7iXl%e7t6%KbVHDTiUZ#gvhSeYJdZnB)PE#iZ1> zcemyX_FwBwym(s^kp%T?F5XC8x`Vct$9Rac4%0a?EF{uu2#|k=_3p8Ryi`Vl*WcJ8 zczNSL`IJ*P97;M-Yjc(uvn-4+FZpYA5OSj_eyES<<`Tt9B>t^%#N#(zdC6i@8^Cla zr;XoB|M)xgg%QZW;)yguRd3!U{j1egd)g9dkUZho3`k^$XuNMVs zJ2{(5xRA4camS;=rLHo_i+G$KfQvs##J&5OgTId5+6QwR3?lMLBygP@tybJ+E0>1x zO?!LXu@t6yEF%o4aNN77?ZjJzDU4~ui#w&FyC&`adb6{17ToIk&S%aF;Y*uG>}Uud zXCzweT^)>-^T3!>P*C^?0=G)a%2?UzUfl~I;%S@E03{&q>&ygo#h!!{-zD|dnm%)S z=GQ!nYZ`KY_C%G9j$#PXvvcgxSysKJ)>X!r(oL$=fy`tU|fIVT5M-3Cpx?sVK+@H-ey#sOB zA&kO-stxdd?Ck6)MO`jEMxmIZgA=GpNT`Bj*-&sw&&?1CG^n0?lFr^57$2L6j)`e{ z;XATU?f)^#R&J2-f;AbR@F^6loMFc|&w8nEP+fhYovbs4^N$U6NI~4u>XDxP)5|qiYHVouA*BE_Gks0vM{fPbF~{ zU?}SS(mh|*teZTUdwk{{p~qP{wL+sJRKrgrR$<_Ho``~MPk@M!6QPkAjYQz;ykJUy zD|>cB#IG`>nXJnK}Gq`tDj1^=wcp|AZL$)u=sI>bhr1*P0ZeWN^*1 zs!yDY$AK<2rqnV|WD6SmEEq_}^yWV`M!|24ryl+;WWwdcgt-2{&bZuPZ>x=4C*40V zdA>uo>j&r(nXx{`hxwlh$a01v!;{}d21Rxmx8p^7c?;YlNF8G#fU%FXfYnMho%w+S zFjkeOTKkRJ!pZlNyhWdU@M-+5iPB`Urma{$C_-IP_K3#_dri-}opJhV7%r6_U*!TB zc;b%moA~Y>V-e+)t!`4dq3WD6=6|1b1Ck*S^g$rboH_H)lTvYe4X&rF^(OsJIBv5K zv=UT_o}O*QBsdlKpKU%eKxawsN8cq%dYgCCKO|?4Lh$V0yx`Hg*|zzfOHY>d^_Gz1 zJ^6|8$dFKpx-Ft%L=qJ!g^`KTVNCONz_N;`F(yD@STYt2q(DJI^wc2*PJNSNDusr6 z7DOvmo#~AupZWezO5mMucEW0b{Xb9npx~lGgvE~%8uKpb@-R1N@!g$cRe+KX6(2U> zLU|=Z`|Iwv^n=^vk!4PO&!~>hkQ811kE=@jxqec(=(}&#tGh>3xHZ4@z>;RiP4ccM zlxT9DA76fKayB7A)yhMkx=7LMJ<}~mLuXNSU*AqfNq{Y1?O%P5OEuSeMKS=cWb%jN z7DJAnETRPJ3KBdoE)w1Xj$=2K7l8IRp(8m{PmR1~vYZgCMdPngBWz-*a?=|Op@1t*0j{*_ z8cKZm+Ob<$d*ynXsfbfoTH(TNuY(u&cI|WDtac`P<}W|vLRowGVK!qCHUk1smM(qk zmgnEiUyK)s1S$0|BuY@8pGL8c(VWh>y*`zroXpVx9y{E5r?mD}Fp4XIJ{HZ`k$^O1 zk^EoZd_VCUG4N{M{4RxjR)oUSFWfb!Yq^TML!A0-*&o45gS{{nmyQ>A*sHH1n4GzL z9_cuJg9Ju$PL88ynpdv+k13(@@yxK!+tCKBKdwTW9vkBe^pLHv-c4xu6v=snbxe1`N#ePQvmMjab z5eg3d-HxlPy;Ab3X(xU4q0nnqgfvU-P+xx7ESD`ardWXAG%s+|yT}zU%yggSUu*FL zQc@_we4_}uO?X}Dh5l_d{fFg0Pr%+(5pxMOxb=*guN&Vu9i}%fn9?IC%my`GOnNt$ zn5Ctx9h^w9T=2@g^tKfWCy=u_@5)`uko$!j7=qKH^zGb_TdQ{HizHQA*S0)dYx7FNz&f+Kgn7YILVn(nxJlc zd+va6dA204YfVjFU#4z?BDq_u>9hRt={AXs-3x&1)K-la;UY>074m$@9}ph_sJ`<& zR_W$DQmMmjW}qSm7Z&mZ;FJW;l9JmH0V8o9#U{jSYG?#ry!Db1YOT-@6aL8*_WmO% zZ-&C)PuzQRU%bj>X2?ilBAqeCW-}VeKyHa#%4fduor|tBSI(LI+>*&Uh zA)$(fOy~U!w&7tnjgq=A~r8&7= zu_9+~ZXVemC*?G?H*r3!v6Z+pMe1hUeOdMR z;{1yiEoQ|?Q*-VAWt;~!Ff_-70}&pc{I2Z{Q(h`vU0q^2x=?7nVZLyo;jKtd$wlW$DH7jCF+S&#C86=I2fV7zPrlOrK`u>8Il1exs>rp$> z+}wLerM)Cm1XownkX-;|$xRZ8*$pqr?q??mrCOX)7xyYT!)qn> z&5J-wG+RB;esmxVZu7<@5EeVU#m_TEyetW|lbgWu&VO*iO7$JKNxV1qq?ActeOLRJ z{h%0JYqSF8ANxL`DKA@>7tr$cRmBW)chsyknydNebZevH)Ju%Eg|N>qKkVd7E(70g zTXC8p#RXvGoI#!yYW6B#>2z{|lsGjVtLfJW8={!1|>KO2K#FS7Z zyxi*XgsokQq=kIJ|8#LAoAh#R4EfO@UaOHrWU8BNdNa5_;uzWU6vEw+?G z|7jSuN!dkKZodOVat-s>Qas02i5Z|x%P;c_3y!m$Oo)#lL8z>x^bpj{kU$WSl$4zP zR>Y2JtPZfYbJ<+f>gn%4i7+-ceqGton-3g@H=svt`6~$BaG^xwW37-bmwIF(_8mP} zGQRhb1*j3uFDe6L-(aQ4sHOsW+^SCl*5$P{}27L`yKP5Fva8Kv(?}P}z&cT3B3<}Db>2x$|uh0JELuNHoYEHlL++M|M%g{+9Yx|gI@3uCM=-3HI1J~c^ zd9KZuw*rodJ3Qofx~?)1{UZJ^O!9C1`DQbW9rtvnB|kw;ZgzGfQ28H2ix^$Jn0o+# zoG@So1@r=pCqg{Lw2po1FW2HzWUJcjf7@$yhZj~Ce-mp2-?%9HTVJuywQ_&Pw*D$L z=Q{5Zo>FIY5HFH6FgQ@c&aV)vxlXXOniT~{*8`B3`1`*X@BlF=whzNs5?4@qEILGJ?AcpZw{DXP6hO6qCgu6(P=QK zp`z0L-6u0EH@5|kg(-^xx zNE;eHVr@+{@tEy zFMOhjd$~>>$Oi_!(E62tHh6NJ`!@D|6Ww>3#rl)`l3f=24M$6`{J-uTMO*eb|#Ra^tAoTuJ1`hH2y#F&`DRWB*2nt-vfMVpTc%{ht8E|Me>0~dHj zphCW3ocYr`{qtF841RG>KaQN%{r^q|)HKIS3mI&x%fknp&gn37TE%%k*`yEH4C+#8Q zf4gCAVBj!(xYK0Y+7IvNE%wG6=$6}_`1noXpW+E7_sF^(T{Kd6$gJK?=p}wc)%ymh z8kYt)g!Odf+iamaEjirT`ppKB=fbfF7pKcWBw5c=SllyUQdF9D=LWF0&XXyDcdohlCa&A8v6GzL_Cmep%YT+~s8hKsEq@pr;Rz zqP4eg=kPgkF|By6(X;)4M)5G>PMf6;VO(p|3nJgI!DXV~rnc)WlJZJyjnC%?d<%GU zD@t><4ZG*^hQ`q8CuLKEr`N)VT4X=JpZ@yus5V-MC8c;YFJo}f?-YfI+{C~*1}p#q?K;pAdJL-dV})g`zt0wVa%lXu0pDV(B?gj^fiNu3 z{GYG2-w#J#P?V81|4ug^QgPTl#`vd2fr2TeFL;&aDWtUz3#v+L2Y@kw0XtQD&;Q4O zOtn=%c7WoWr(dCgzl$ zibtv>$AiU}Z#e$3Y|DGc|F!6YH2y*fwf)+7Pfh^Woyge4B=iNz2%8{okpINB7J>P1 zY_y))L=GccWX%t+pkoT;8s4|v%Ixz!2qsiXv9hoTQr9HhW6>(>AbFvgleQa;p=khmPI>Tf#RZw*0&xmHhz_x3+7eZ8nJ|i_Y!j+OBr4> zowHe*2F482_V=<-rQZA0lb@~5z4WQXx$DP&p0{i|H+4hF->|)51c)oNPFEJ}Bed^D zJ$XVCFg}+bg!_|!So@fX-=YhLppy|RN2Wk~A42_r#Cu<(tACo2)Ax-T7wnNq)eF4O z{j`#;mX*Zu*Ox{Hbf%?`4;!X)V3i0%P#>pf zsmNW)JK#E+fcM>~h(e=fQ*oDp!24NIaPZ*iT^re-Y2A(@A|&5-wFkUhAKprnbXZ@( zFH51Z;<8|NU+7o*ktTFxoE=`WqVXbb7!QCP`5Fb`z$TfT)vG>cJiPtB=((TF;jKAU z5@I)1(d>6*U6(JnEXGfecb-#}9$_lVm*05Tt#I&z=|!THX%B5s!-WV1?x~>aRYDg* zN}aVi-xufl(c!(>`B7q%Yh77gCEd4C;0;I(Y~w?iscQbUAVuzPav3imktYzWM+yqI ziyP`c)n2DdYeq#rEp$p5$F+}0$(>{iK`?y?xh&VSV9Ycg){l1sN6d80VQgS}x zHwQLx!f+wLubrcdrrUC1?V!CP`8`*2c>6Q-3m{DW{uIfBzXd@ouUo%>QEvDfy1Gpo zLksJs{9wZJ+HJL$IBM6KZ1lWCJagM>qyiLGh*m~bhq4N-`31wePG52)oXhBk+Q#SirXF(71iwjGam^&b zIx3h7;>Pd4>C)Fo-_0aVxBkrJ>>%wDy-e9BK~#vj72X0=~Bh5O?+)i>e-XHe6F zjsB9r6(NF8%kT)Oh!>VV^#$ASEM2He@g|{~V#T~8;FCHl?h}>^VKsy_zE|yfBPf(r zG#(G_d3QQo68W>%5gEVB{Em}%ngJ?=)Js`c7au_>=`9==7uOa6Rj!zr-yAmQkRitR z+eg@ModY!oM7@fv>}(wc;WX(8G;=x`rmVew{7Wf{;wrRQg^z(%sBM>CxE2s-QAE^)II}ww?KodsNv4z zDi5QzQ^FNu=vPJyp+N~J<;C3hE4N5yU9_w68(ut9N?~inu)%hZbcm0OxiOESfDG}L zfLACG0H8z?1kx?^zrhuiG@zyhVfv!^1*s>L+I>=y+^%eYZO(S98X` zB0gj(ZohT+(!h~UeWI2Sa{g(-$)A1lr87NOpogv1D&KoTQhak$LE&9r(WITy_iGMd zrl9>b34Jw6y3T8>Jx?#Aqg21yUz6oI6FvMwkhPoXbT6sfD#VQ_e)luc)VrU3N8=--eB?PKC$%@SE`vwIygzi$ zw8Rf5vvGYKgo`U~EA^0_J&m5hLfG%dq;C&HE1DlR1k-y2{zd7R9?1f-)u8z1 z9R$X3#2%HF3NMb;7%aMctIC?0((ICALIwjN5a8m~RbmmcRwvK7sY?qe>}iBV|dX z8O!beVqiv+s8tLbq0pJ{Qe2mx z#`HIM}+(ICXu4XMtzZ z6Gf3EI&5pNoPO@$i+`b?n;r@J`zs93QdETO{oOW6FY_4<)1KtEsJnuJhOdy*_0X_q zm7yVCoy^Fm>8h>BRiOIA6#>#rSIk0TeJZKtw4f3+$!Hds6D%$+-hKg$=Q5(U3s%-1 z#7^B`)vH6^)SvOpPiNKO=byJzHP8##uVPoyc!{I%7H)7|doPVX4;hoYi>*soYV~;v z(owmI3AKlcL$d<+#N=g?vurl)wrM5m-0sSXi=KTfVEAJA1e$GY%5t#aW$TS zee6?cgryD{{B+L4xJ}z-lYwbl@_%JEv+TG&K0fxMSE6oCEG~xR=v5d4r1wh1ndR-h ztZ!6N&>ZwhudSUj-SxcV1OQgx~d_E7MsP;P1nIo;+->JBrb`Ror{WT zR?=}I>6|m1p+`jL`L51NlHM!xRk>z9gWHeBN>}P*nyb5bz>{(6fLVqy`UX>kjy<3M zth#W;;M6Ew+1{)nhXZHl8`1*4d;3rddw^C;08T%q1y{tJ?&WgalPRs;v9k;RFzo5+ z4{3~L?8*L?ls~v(;>(4`bN;e*t136|v<{V!QXG0-<1%0ACP`2>SvMA)ReCTXY!zIP zq^^!2J!4v|7%p;&G2-+1pb4$_-rc||`EZeLp_orAM`+Z25zl;~W!g~IW+F>txuGU% zBx0ZxNflI+O4+dDQOSjUH-sE6XqRQLX8zXLK=dZ6_LVBK-Uk(pyfRs>wNduBA{tTjyTOhlEIXLXMz3Wka>~VSV$w$cUBHE%*RCS>WljAX!#}gS$m~P%nR7p8@sq3z8YPpL)#@ z`bP#I?(|R>otN?{#VHclK$@A;0&AdqkMrXT{S`scLmq`@wQxEi#7$Tz8a)g zvLxmzR}C7g5x3mfV1kr}>4(~|#P}@*Lyw%v@+tetXQ&S8tnR1bbbkU+E#~0+#CF4_ zBvqapGNz^%%)h*TVv*=`zC&`41M_E?{k^_CLK74eWSj==Jm%nX>lsrsFMZuBi1%V# z)UBHMnn85o9u<^>-xi1EZQ(saS#a-Tw_6QVRAiT0E8X?DM7wT$d2naISyUy?AI%{P zjA2Swc_GwxzMhtF>2xLR>ey<|-SJwWWwJ$3h1lNJ^Eaztcr2H7{x6vRLGXMRqc^lv z@PINHt&}e25Ca*6pqe}+0~rfnAiTd++&RdNt5o>J01rc z{a6T0JIqT?9VIojC&Rs)M zdyJ8{FcgJ}^t|5S{{H)`pb4U+F(jzozkwU$4}9#$RiOp713`-d+s3V@dipe;{sqvr z&I0}m6{%C0ZhMpd96d@2PktNQ+ykZNzP|H7&Tj&Xi>V7^dN_LAfFf`S+E-N5G6Ln) zoR>O8=`b18t}z?pBXmD6N<98HT>M7pr=yx-)75%T&6*!Gu&eHM!_KYCrMtTuB&C&>mM)Qy z?(RmqyGxPoRN`A(J?EZ#?)&@vL(lUZA6$Fywbq<-%rVAX-)+ez2Knz<*?&$kJ5a;B zTpR$b7I-^i)M_q%b$nghpyxIM`Zx&>IRVWf&OQf_Xv~c|V{Qv+#aeH4Tv>)4ed+WdkljW|1_GyCYrxdT*FPkM4X(MfQ126^}ppNa&wc@+`b^wa2QVGGUGQ# zW|2*Z@$D_Z0iG&CQi2HYqb7x&+c`3=lOYM)#VSaL-GRX@}J6WHnEn-Sf&8V@ruBl7;mdhA1d_=R^~`xc1ekpM1gP4b7fQL#lWJVQR1s#z*pI zxIYk@s{wBBordNr&N~ZkEsK+%DT|6!;DCRQ|Lsf69V{Q-wP&nmgMad3KyOmP1Yq!TojR(96>4dUGL>~BmsUn;hR~CFy?E^Aq zB{ed(Cl7(JE~3|pGGC7LeB z@_=6fUWk4p8Da*V?{Qs#!`&h1^6P`n(LHo@f9Y&tYikC`SwSL>0T$*(I9Vvbl*>k@ zTtPo~ARb^3NJ>uD0$cg-^)e3LX=LhUpui*xyM5Z|egdpb-SjgnM2 zdgsw$fU2s*1`)`gbYG_yxqJ_|s!@HtD_Pm!CU42-gctS00hgJTs?QFP$|By>-2-&r zlU1OK_x@z->~hH+92Ji=-Aj6Su`zLRnEKO>Yfa^s+hA9!&5pR*WV`;P=>M$v*5^Z1 z5-6Yjm9-Fa=h2mBUcx-EMc$eGx?L$MfsL^o8l!Qd@6qm4ax;M_+S|>iDm+dE&;3pH zV*XK%`Go5sAZqhn9};w18G?{bq@IHJ^!yg_Hd*bp#4+`izuus4$``5kIiV{*f3x)e zo0fsx2B?NX5My4ANelph-<*c?wu|5^iJNhszvsb z5e36r+z5U?n!ObZ7_+zI;>Y)BpX++{Dt-L|0VKFM9;K^M*RF?vji`FAfNkwnQ!Sp5 z;14t-0CHJ19CVFVtS&56{KMlF=K^=Q+u05lq!TF9l}|t;fvf)n2-@0w&V6wQo4lvJ z%IUYwDLHq)XvA1cC6n&h?w4p^m`8c8836p@1TB?PURfkI8%Qz5B^X)&2j2BCC4Y#5 zZx&zfdQMqb3-EWwYN?5R|C|-W=KvW2|2+^n!$Z`e_zj+huOTloe=b|>#krqm)@@bO z%?6<+4ILIOHB}$mb!pQ9(rf0x+oKuJXgsChh)BOxyrVxq$|?~cMp2JtRza8c7wfMe zNx4wy^fPN{*f;B+MCz5j{e)^+&t+8RC##dRC9~n07Tm=%LjvidQW{Ax8rTRps(zTD zPb%h>Sl(!xvZ+S`hAODh^=9_GC?A^eZ%tJGa)}&2Gf!onS-#GwHWlfVlh}qjFn`Q6?m~T?g?*K;(FP??Hp@ zPo|ouBqF`Upu4K@+@@Bk2ur#Jt?|DFS^6>#f4#>^Cl$CWc42@KduW{bZ0GN95DV?uH#Q(pXm zd`|u=igITbr93=Fo^f*FI;WPsu>`bIA|-y$**Euxq>1RX^+oV#7@nbi|8Q(~l}gum zF)1KTN!N?7<$4($o$xJJL1m@eZ^`y+n@vRD&M>ewQX=9aYktVG$Ev-<2-f=GV15~ei=Gxd(gxHy0Kid0r((tD?y;B8d0*)`BC1zsvJG@)kw>Q3=pC`<0IcyL;c@OSu-_R3~Uy}xOd<~bCwXUH`{_{fUf@i7; z0FT5vf3+p%7$Nw-=#IqT#D}%D=YM+AIG%2y^f^w}#U&<)wG1891mRs6E#(4Nh4?xj zTwuUhM5G`QDtTj(SlZvpO?+T#kd7|{&RXn}45g=Wg?K_xe$>!&)V-^FAIg*|PxYzB z6O)S95xjw}9C%t$D3iWHg58q~L`r!-J#2RGX!)G%*X2nCxYuV0D{X!#8o14oLTN3) z)Xuy=8dPF!rV~%TTK@#7>g#F`wBnCzQqhr)3C}2y_pd8SMR6z4N?GxPOV+-=2?;^R zhwg4RL-_}z(qE~+dEam(BzZ~+szzh9Wq648O7gMfq9MHC^9G(xpFsJZ{hD*t#rTf& zdXbowhf(=4W6Z~n&!bV(5hv&k_ertF^oaC@veTo6V8kCPR|bJS1pSgrGCV_o=4&se z;^5#Ahw8{YifZlbga^klp5Ki_NJvOBks8C)>)YBn2%wL_h(x z&z?1JC1aKf%e}p|l&=k4IUm^mJ~Kx%T)&2Tvd0B?n#hw}TY>gc4T&qV5kaZ4b6{Oo z(-BrfhM{9J3T0(=IUs&`4hx3|H|1LF`J-<$tA8#n{RjYSrsN=9-8uEH9~xgz!3UlN#cB~ad*Ct88QRJYTxScXa`^_1*BMm+< z7=uhW(%hOUnC01#)7Udd^6Z*)8N*B-QqVqy1%^2TGIe1Yg~Y&Hh`Oqs%>ji3MWc3& z{+r+fye5oajQ#&=(5CP2mjgNlolcYH4jb4yLW;&@K8<3Snr%NeZ@0$rIT#irW;u@F z40A~aQ+_OzilIJr?|tjb2$qI~fZ|Jt99a+O_P04{lOsT;BBRg=`l+JgnETazAam}J=oeQN zHbho0M89>jcq~mmq0>nh^OEVXiNI3T^A1sE6(vKI#NqH$;eyu>(`YSdig(z;OtjV1 z;yo@^^VfSbIW~3#uuId%hFMQIYPc$zSt)vW4m^HSIc*E5A!VPTS^t8)o==1|Aqb>x z2b7scpkxMx%KOB3sL>y1(H*sLA4ELYQpj>n(TJnmIO}#^ePGr~cD0I8AmI9f@$Bmb ztjpTq2xR9|K* z%^i{Qkppxl+xlvKHnAU?Km7M5kgB)91P%h{vFr4zB`w_vvKFPkCQYHaZ|Boan%ESzW{8g)Tt}u+h9A55dW4#KX}JsO8M|PmEgWgb^9QHONfr& zH{R%2pO^c$Ir#QYCBma^Cd)gzkEP{bj4oOPNNhc(E91k9t(F*&Su$_WN)0=omKL}b~%Td~DsYW{@6-r;Q? z8^&pjsOuC$xCe6U;Vtv`IDOnl4L{++lBWb7@9x%jEtg(!0c}9onWnHI89>T(*WQ1{ z)3LFV5-PRG$tpsnt=*1{u7oZ|q0dET6Ye@wgMZ!Br(5mDlnE)*>>`G{{}y)woyvSi zM-Yw;Pm$u2$mHaZ)m43%7BFFAc6hA7rrfV8y46h z1N|lO#u~zxYWWe#ji6IjX8avLI#pEvC_r5D1{3K11Cbg@nbXEF_jn0dayM=hgnwU? zh9e?6g45XMl|(w8mk%L}YgAI6`(SK?pX3+voA9 zW9-X}397x%#Su!1dHA>MrMdC50jZfr%3db{%o_Z z$pdpinFV(r7!|(oV`)}8#y!*CAbbtL*X(qO7SrkO0ykcxGR;_ZVUfvq>>>|HwtpRG zH6r_S%>w-yZ%u|S$bb#djP*gU`~q;#I;BBEZXF38LEVwG!W(Ik?h@k2pSm4o)l<~QQS;q!Cgfl%2E5WA-=VEiX=nYYjvZ~LJ>9lNn@FypQz?87)^$C{AjNe*c%?m zJ6Mif<<08v@SpQH&9fjKdmesq<~3dw|FWM`Sj=X({wI!2wm%{ARpCGCriSr0>WdA6 z6265ewHX%ev1xf_e5Q)8a(a&_-lEUC;({c9*la6JX)3K~r#G+lLv&$I4=5XpYeWsCr&+AM{bS_bB;S)(wmCc_R}7u1(5;g>E+sk`=mk=c6<` z@rUxhm*^d&*Lce*gx?QmtzO0x9P8YzskS_|xAJ%xrl}gV@3iPw+w8KIPp?qIxGSbO zapdvtcV;%rnQxAXfQSeOBd?$!3N^8;V*nZn@ZM$65(4FxqS0evEd_M&YQ?QAVT10= zYlwl<4#*h7S}O2LMp))YW0Wy7Z7){PbNlgnuN=m;2H%F3$K9Y+5Zx!$q8`LIX-(d- zJZuSEL^v$w*u{rZT173n0Hfn`WNUQo&vF)}px{tc87KU9I?D^pQGhZletw(&%Gn%w zG0_uU;itQA*rNZ_OyZ2maZ2v*Hgt2>6xtUP_^H+;ekr*+=@mcHgIIxt3i?&Xr)-C? zq6C2Az>pB%#zy|=SD&XhE8O^eBE;j*8is1Yj_Q254Juq~E(=+zC}9N+zby$9=xG#m zPynLhaOwzQ#Rkt!x_491wDYUJQZYO(oC@Z?a1iQ8Atq{)8Nr1&irL_|VOBH`)pp2eFrB*5W70BwPP#<^0Ezyo)j_cqPfcnkH;U4Y%! zDV@Clpn32_H<;;wVfS}kthbHl-3IqmP9Xx;NDl#=o6Of5To(@SGHf1SpvTXcw##Hs ztWF~u$f~3_hw6-bdHP9oX|MlyUGSw`;`R$;ebkBmIeKts8{(s&s0f(dM*nVykc9_# zy$?)vIzrKehXg50N@ECj9sNx6*%g)C2z$7_sA*YYbK^LK3(Xdg2Vu@>I{c-DL7chk zda@iKmGjc7(jNqxf1LS5rzr}}+SlKty4Z=(fXiX2)2=#Tz+%80EV}Kl#HeU9i@7@B zK)T2F08HV~Lb?ja!*?qEBB1I|N{*{{L@ML-1a23byl5-VUb?Fxg+mF!egr>eIbE4~ zYf-!Kp1}dz=g6`;b`aSgDzG8Jw#exPFH-a9w@fqLb#UO^vZz*e%SoI4@e26z1!fi~ ziT9`l@Hi^vue;ImP-V5p80(3pa)E@~Q3x(U)h&mTN`g)ha9xF#6TGgdVc5$<5ylLhHBvNVi6Jp*$dYs7UY6Z#*dqw36}G+YIPR9NpHFw2 zp6wg4Fr@(=ebg1@*)vrYmP1fR9jRS{2r2{dUUk3Gt@-uGT3sWPXd|srkm+$hMJtLR zO8I7_B$;7942FoAfWiNtK*n*e$Gik&dZ^?=ArkqyT_esfJ~fqzGcMW>52nVkV3YB>IN3BNk`pDNw)BK|gz(wRY9hhPNPN zO)VP!wl$ELI1HS?sPTsu-4Otg)SfYWVDxa=MfuKa26CGTci<1NbtMK;(wgsh>gXyJx@m*{Xpk7!8n1N&tSw=JmiS;4# z|1dBYIPOiDvzL{X*VLS_=C zB_u(Xohgr0cZveen%>4JFAPgUiG8h_aG^pPT}w~_JG22nTU{_;q1xOt(1Z~r&B9>u z2hQ8(>2y8GfAL5EPMeE}d;!ZEI3RUFNHVyjGxG9Y*=VN8>VkoHE=N6&nE`+wS(cgg zXNyR#eN{P62V}WM)ee4?T!|=IWs+WbZh=_AC19`ikvJ_drf}QVJ`=QAK^$o*{Q3sdwoQ8j_i&` z;yqg7U?V5@F9f=VmMI;3Xz@~?-6r`_erzQEnBPkDl-ul9pB zfg1gF0VTGAqbO;z=1$0uMDi@$c4#7IdB) zpb-YV(()M(PO<~aOj>TQj2B9id6&O3KYQb`=_1$1C)Mi&^$w7j`#wilg?^(Wkn4UbQ) zc54WKl2hL3G|*W@MP&SV>yQB{{@o5yOe?0E;v^nolU!}Q^mVNdj#oPd@Hga5y?RiE zy0erVmL424AOo}fRX?DKnwN^<8Hs0W(SY_45zDj67k>7aytlXWZAw7+S7F{%-pJ|d zD=7%w-$(@T#0hpu!-krF5jXzwtuez#YXgDCaUY`Rj9YT+~lYseqJJ`25phNp)>yp<$21{~VBYVrV{2I~WfE;3jBD z)iMwQTaPs0K2_E|+nIs!-kqy9=m^Bhl8jRSD%Fm#lrBr~K6rc5#23yzS2tuR3 zV~h6%)7^mW{kB#j7IJXK&ws%Ab;TDb)hN1xWATlJ*8wF2y5djneMI%+cUCOD`#+!J z7_BXU+>>C*>z047_B+|t)$!1POwAxj0uHu|n^%FMZEN+Cr$sty?&oQW2iX1?lv$3e z<(oN!Be$@Yafr?D#Kvi$V^Zx^gZgvOc(YTx-r&K_Ubn9d zuP*(y`y1!(Ib2UoLA!?IaZ3A+X!B;L`@(%Lo?A1({i8B+gk&qgvNkkxL5-_~n-pIf zzuGh`GzFE}M`+l}8$*a{;Z0L@_p5w+P|*`X@xSjDQC+LADzY&z!7ku2_M^6j@ zfyK!;lQ;t}eI8(0>%H~94#ct(=eAFWksG{aILQk;QpuF;*?Cn2Ju_ZwAka$d%-#XR zA-uv>fI%xY1Of?XQw?W`wIA~&ApBoz-`(AnQ&9mDBqX<=lh>H(RaJQC=;$To<-%50 zg+VNtvHHJ5alH0yZ=9S1mmnh^%MJJ4P*2Is5W2Y|tuChTU8k<4MS{ZnKMXM+9F7ZF0p$FQsz>HJ#kG4<;uEYKp-%WsJJ^Ks9IOX$)6FITrN7d(^^z zXSUqVLJ;-?e0k9q!QT^?iZ=V%dHye#EHkjhKYgOmY;eIc8Tu${XlR(f{*m7ePpi&B z5H!kPzkV$lP5E~WIC;WA(%HiZHu4Ui&&c-Yfe7(SpHbq8$@~$EL8|*IlN+60NvtT; z>1`std1heo>so`wZ4gnww(m`J98FRxOv^$S(sh?_+U|goK`d)4Pb#?~7>j?}DIF(rw z_TcVEgV~rFW2xo=?U26J`80eIcb?78Ee43GE(o2*Gk}P*DWU z@pW@sZ5Egh6T9I>m9rBZC_<;BFZcqwQ7%7qEDm;)cq1AZ5s4}%B?%P_EWe%sukXO0 z{_f%7ChKdphrCSba|*W3$mSzguZ_rVxXbSX<38O*aCw7%EjS4XPaOLazx;*`#?RtG z_Ze6r9dzKxw2Y160OP=e^K+Y#2FMQ{7bB$cNC0}rdd4c%*vb#$V(2;7*A)+uUDvm%_X-XwX2 zqA^MF+NoTuc_ams^AYOeTZTu25$e$GlYepV7w*jp3y=F_=umZbWdHD4fvKVxp z7aWl+<|H7SjqmG>i+q!H_whMyg?WrFXsjS~1Y~rz=uXe}NF_h4jQ)S8$64)cC#bb( z4Cv1u^;YaI|GNWSafv_)V>w%a1Qhn~USzxl*a1*tf@#UdpP+rzsVXo34sdu@!v$_y zfTd;#a6;%dWC62N(bv9&vO4#U-}{rIUM62}nrTW)j|V$8Tq|8*)XG37`tque;G$Qh z=4P%wRPuesajJi6eYs1}0+gorpRz-M2+#oOf(z-WyibE|skH~sFm>+XQ zFF(tJri;%l90@7uA+QY5TklU;myEK0_a||9_SwW~?c|nhGMM{^X!`g0O0=au@g5sLs5y&JSY+f?tm5Z#_5wr!a5EuzRTn+#9sTh?Q2#72 zR<9{@+yGHF3|MIk=Ucf;Q0FSXpR=w<+i&{G035fB*HJf(|3uyOj4^ZDlj{-L812zu zF@a-5Fia&dI>w0CmVsE|*~Lr7^)1uzHI&r1ZHvm81t*Zy&aBAa++quA_JaIJ%xU%; zUSn2aBT1vbKb5s5On~-JBYALi|B`rj~`Uy{yuSDbCy zcDemAW!y{A-Yp_1H?Ni#&f{VKaQ3Sa%dUuniSm&d0-3$z4!CvPb4w9QsZ>pA{1j%<&cBB)TWcoE( zK$Y>QvVtm6`-Usr4S`3f*DnHEC)!tH_aw5iC(I+{9AI98Pz1`sKBa`hL>rs6kd5V+ zP)7pXjCwX1Oj7AoYyZCs>wL_yGGAtL0_Gf)270=F28CYJtrfSgir35W;M1-9-l4_( z4}R1tK+6hI5(bpmY;>&-Foi9d+X;#O4O{f9hZ~cOm~dbG(AmyNN;bM(nKC6uL)MK#fb`Hp~t1?FL%%hTxfNYa>AcQC%9_Z;Sf2q|F`VanbS0z zjJ+S7Dl9+wgk(tax$$9|@<7Zh!@v{buz2(_OdMBUK+2bl{`}^Dj`%+oh9$N>8*Xbh zCEpl-JB&DdSQQ6oMn{c5yExpN^is7gH7y^+@_WRI6Kb^r4CYHTd*+0x9S1ErB1fuN z?^_a6c6vAMkdVW6?+aUzE_gv}YF1ZmLru`g{p<4VTY^uKp3A!=ZCt|y_!HF3>v}jZ zqLzsM?=;RIt&Kx}smbHwTbvFI3&8qK#i#Q(8f0>kUouh3p?TuuH- z>H;&emiazB`}hezinlIph`1i`*xrLXsE@HwWdBWShPM4KVly^U$^P|Y%=PcI>e|rz z?N<^8rdwPM{BY2-=vj*fuLE-tVCo(Q@E`PMOqHm0u`hB;8X;eInjs)Lo6*`|9k^0n zJwG@715n&^2*>jv-j4mB;8r3yjQ~8PC_qX9mJI^~hQ0u+V$L^>(%J10Lk}9tV_vCl z(?2qTg@Q-d0y%7yT~_~rHK{&Ece2VQypd$Yd$XT?@IGO<1J!k_p8dkgrc8V12CI?z z_SxZ$fB}dG275#RlIsS;u|PD4tawa(`a~*t-En8jujazG0~OsbrE$JP?E^C)&I0b$ z3jkwLK-{8nrNYGhZ}Ui{w@Y+8%M0Z4K}H|tdj6fD>iLwQxq#y901U=y1%nU3toQ+r zd!Qv|78> zH*PjY9~lqlCf^CU*speFb|dBvqjw?FW{^(*+5vXhi+-ZKT=|vhB1xz=?hF}Se+*M% zACYlhf=afa4522nuC3376Zf7YBe{1uhMrW4|f(;)9q;`*52 zU6vROz7)`5f?Eb^UM$?{YE$0RO_$X8B5w|~T2MIs3l@W{42||C^gnNHZ5<2?V+dxC ztpFeJw<#%F<@R9c(cMC26yS~Rel}y`YGMT@bGQRaWl43ln357MBo53NdQ4>=Zs-(t zu*Kd(q3uE==xbTf^-I|xQBC^7%A?QolZ?oC{Iadg{H38lL!&znm|PVQw$K0$DHyHj z#lRiHxhK=|Ugm7&sz=DvbO0IfB>-Ihwtn?l*H34oway+yEb?nAcTFmFw&oW~@8LH4 z4o<7)n9uPU-RnIN&{J;VUn$l8)m$=no2Fn(2Sq+{woK}m=9^PSRAP<-QG4Lgh5}Oq zrri=-(STVq24D^+jJHz0GNH#aEIGGtb6);}1qZ`A!)6*zkCffm8r&W`pdF0_^G;IcMF$Sgw9r8ssootrN(ACOTySxO>C`+m_`+7vRk;!9nV++!w2C0I;}d<*96h{O+3h*Ow3H3745g<9fL{r%>{)mL7?fC+U@nqKIoQWas@@19?o zv?V6dTOg7Z#^AqOsU8Z(3$-pRG>RSr*(R~)60nDQR9w%atfLd&-!BPy)|d}Hz+@UccbZBNrRMqtKa6aGBQ4PHN12?SXqc;vRIzlz~jP` z4{DgH1vB!@E^n8CUSu_9N%-(4@!7P?s>j!u6b|7(4dls;%iFeeV`$zpm**ANBkAP8 z`n|gR&g0&`;*=P>@~rfvB#jJv-$9$f)9-|f{S)n)K{<|3o;>N0PFsj_CP5AA$Kwgj zqf~xvPK;bWMyYb(y_MBZG)A{O)J#Z--$L}WJw;QWJH zVxLwGS1Ug$`y5519gr|-0BC1k3$AW$yHHFvL!Pq(J+;|4I$1~r@#nnZBdkbU`GotC zrOyIGP6dtB=%sg>0@vza%c@U1l#6G>jmq4oycWSq*0dV2x5d;CfNvce>Iq4Rl0~nw z4n%9hDQ$HI+!I0YY{_*M=q5)va{1Ob0lcD0X47x>6n6N|Y3E7a4G(xx)gej><(PPh zKV)HbhyPvB$AW|uwnCl$=Cs%=t*8i{+~Wib5b{7VJu*_dat_ATN@-^+YX8*hI)%Pv zn6&fpab}U{>SVi5zGa}VE?STT$5*oCx+cMUpID6e+aB{hbTJat__hK&&su?hQ+*nt zeHC||dy!=l_sBSbg&cfNOfcY?jO@8-HjHFlHO8m{vMn&7mEIY6`(oi_#%JQ&lO2uJ zw;tQ9n7|Mf_HYIGq7YO$UCB^yufhtX{$zTv^(=?|3E*$Mg>LxBMk3tacg7}hwDzTp zYs5Q!7YKuAiyK&g$^l5O_ATPToF_4*I4%fx3N!Md?tl8CySTScRlNbx-vA^`8*|l+ zA6>U{m4})=HHHBo4-Oz;#X;K_9NCtK1>`%8LUJ0(6PWq*y&~WIGHf@;#X9=w=9;_C z+Rx|Eh20hcUQUupcUm}yh?K-7xdZ_x)(0`KFlWXvBKfanuT4oI)DECv-9t^6XFq`T z>MFS4x!kRSpHcJZK6mH3!*ldzW)?U(Zjjv%-xEBH=U$dDyuT;*D>Z%9@-465_`Av^ zo*tO7TXH_{3`PbDf-wtF7Ipr~uB2Kn{^pNB02`Lr_YUO7*b8O7bf9_vmIVc z&h$YDYu8QByFBHEuOvp5$n{ex?O^%=CiY4Y_^QA9#a&s|hmZRn#PnkBaT2U}`Y<;x z-T4~*fD$z50prLvcSo`O*kb#sEzk^*3|`=ZL5{%9J{XW-H?UX;rJb$}e;X-Ol45Jw z{}h*x0w2!yV7XO!;0-XgJ6KP$;ke>{@Zc^tH#e{#MFb{90&F}cpieYiW`u%(M&t{| zVQhpVr7(6(6rNwmK(G_waV~Qp#6DM&3#10U@QOtL_;KJz4BB?JKEWy!u)&NU*6#3?@^wddh zOz7rUR+z}eL+FmhYHgH?ehAc4+Ms#~33B}H6@*6YYt;=9|5Do(p^G0sIA+BAdwYcd zwF*Ge0bt4!1Y&su|2&va1qG-8zG`pvL0sA04Lvz=0AeTt$0p1_4Y5A?4|Q-RO-Wb=`BNo8UAiPK^>Jn#W->)zY zlVpHe*aVp5Pzzi9v!>ECp5l|>h~#5Vs6qT=6)&ff-hxC2cViK*lr#%!V&7TFsGW_@ z@tge#Dr@GFKQkPl9{ea9TOsL3umYMID^0-4;t}?*HF&?H{*@t-dsd1@SxL$v_`-jS57tO)@Ku)lxG=K zDlY)5`{nl~dh4{zBTqr47&6M&lCRe&h#SYub(0~*jQ^(Tn7%QqM z9lD6yiC8k_EbO)pL`lRYyEYBp44Wl+4FV&SjY&>S&|@9`>GRtL&Oa?&&(Z8CJ&6?eXr3}%D}(1U$jKsog6VdH~RG?b+x1E<|OlpZ+()+EMF&_Gzy(d?=xa%V!1Yfb^d4A8cI?!q(lsC0_t{0&03MmHgwN z17d$}9H%N*vdHQG+B<+P`PH}<1q&-O=3rN#{ga@o%Yh6B@cJ3fX}ZvFe>R@Wj#g?# zyUAr=im7$SL!>_h&@6%A8nE7nE1hA0dyAcm8v?rJv+%b0_eq1iV9K%m%S+qWrNT>* zq|mJj5_jX?M9G&=BGWTuylQ%>F_(6mb7IyhAvy;Oj(WwN5^1eag9NWt)%DZ}39Auq z-TPp(3uY8@di;?g-G)ECw;5{(4;ph<(I1RPC#P1r=Y-RWXvc!Pnktac7qjLV%IV{e z*Ldib95op^EW}ZtxtW%|-0S*9sd3l_JY9xNJUsw}e>gMFXp$bgdH_RO?@9@}>C8UD zdL~0d=IZrd%yV+6)?Sq}S>S?cD1wM%x7FAxm>()v!Hw}Zcdn+B=I;;tt*+z;|JPmZ zEPND#Lna={#K+($QPLL^_-L!?W5&IDU<-KF&4IdnOzE62UIYj}ia~B72mJOouh zZNghyFGj(02x}80n{kSKn=%jKsi4d4ozXVgLHGfT`SZ`)eiiwl z9KDMbC&HB(>y+s8$Pr-v6yW|px4ne6!{fX%nT<{!fdhEHjIKaHt&$je`r?XLLN2$! z;!It}k9xB42Ta=)pUtHwgjT5vXDO)Ue*t6xV7V6tX4}yx8W=nl?*D`d0npidi<(NN z@Ya?7VD4umSVLDib_7@vsHQ*h0kqEoYCir=^ZF-}JA;gV_4e4SAq(Grn+J7)>_!=M zPm*5bca}^wx7JE@(tpGuT=vu2=X33w3oQd^gC1Ifn5Q&1`h%=BP$=gVncl$Nhgrt? z`mP30d;ad_E<7<9k5JEO1HOggy{}rD>;;}@Sj_Z}ZO_cr=xT z`+E=l1DPzf1bSc+Pm6mMd5IQ+LH8SV7SMT}y$-RHM4d}pGY0^5_3V|o*l@5k(`n6$;;}2>0PNph9Ne8!4 zBi$oA&IGAd4!JW)Tup1A1N+`rgE4Vjze8x=&P z(mQ>vK_1+o4GRqnfspXMMQP*yj#jFDFSiXC3d6&_0A0cX!wa`2qmT(j{-P;{~*Xl5hA+&c>t zR6Gy?0ze_eV$eY_qB8mIJg*OsS1?r`>HRPP*M+~ZTErq|{?o=lV9sZ|%_k0O3z^Sl zM&HdBa`Mh5*TFDLc$S3w3qKLk-Ikt%UbLNy2Dlcb#q6||qU;@bmcb-AE{_xJ`_^Y0 zoOw71A!t6%pMnWlA6rB#eDF>Fka=sc-?WxruqWW{wbwM`_P7A`&Pxi*h)`5`#pjsw zaN9+Nm9xs*mR_?wTX~i0=WS?DgwGaPgb)m|x(7aTZm+KweLqLUtTm2E`abye*t78m zGXD@)z-0JP)X&E6B$nM50vEaM5!k%sW_Bn{e8vn7(Xd5iWWUs3+=M8;pw`ri5PuLD zUxI4l>9UyDc(go4hk5fpH4Yh#r*UBFGEWJe01YehJJP+gwjCqdc*kAa`^HN8w_olU z?o6~;mN~qUDpwA-T3XkgtL6@rLF>wVgG6AbMiTwxCUGhi^&X+m$=l$_T(w}<2}H`} z(VsDFoB4L1mZ?c-F7C+(kpxAK&BjrM-V=Y_mh($D+dXlS1SDs2oNZq;L(8|d-|2Ex zRZkdMy)CUpN7k{{_h`N7;IbU--k)$35jjE0ReaZh6)8@F6%<=PcW@j=MKZ1GeN>LN zpsf)~l3-&Szj%8nC`q!^G zg={PvzMkT>`pzF&b<;2T+n`^=uG|ED5B5We#AJqloiK{zg#t0dsa|v70d;6#UD^TK z3)?NrabrtU8=U?ZisYBDBV#u93oY~gR;*nodCy&b_A^88Pi|>>t+?$ZKZ7Uq1eodS@(RFZ$~>R05*s+o{bSUBMZ%UR%4|jQo*JJK0CN+aOf#2 z4vtN0M~8+k3$d(hEuvbYWA}K0sF8to_@2r{e0bqZv9870Q`x6i;3JcbuHsQWJbQKF z=67x$@(WJ*>iOUug{JE@R=PA8S_1=fi$%2s@=P9>=eA5HX|=pod)i+c>;sgNR7CEK zlB><#?6=(vT9)emz_Fn&F39Sd>V5#oYTvI3FemE?8Tv6Jx%wkaMz=$_s(#X)+0Z?= z)3ypPlFR!>!0t%#Xh5Q+C5_VEX(lM!D5s1PBLOUSv_M$9Q(D!x2xTB8^7%VKJ)8)< zY8bF7v55IT+|otTe2k@}a_#^hvK=+D-p*;j4_;`C9lYj_6^X#_m~?m~9@ZSQTM~t{ zW1c_ZuMk3CkL1kD2);qXdv;E_m8jw7F7@LR&DFuQmF!bXopEnMU=U7tXPj3BJ=J~ zyhma>=%453V5vVh^+&Oad}WBIrm~Bte9cLR`JqeqZG<>UldlWjz&dBz<)&kP)xqHu zmENU215u>g6Pn^ytJ|B|tOQ?9NvW=<)r=Ky@b0jRJV0k1cP5yX?4B=fy$SWB=`PqA zVF%0`YKGu;uq<)$U~fh9)C67nP^;^#gJ~5OLo7Nj4G)5Z#7gkF{j&CI2F_)>#|eno z^BZ*uc6eKE;66XgE;bD8j`_dV7F+e>HOXbOnzyc*P1KV{rn6lBh6nuT#>AT{eSr9{_`LuKfFu^=OF_Bd!;jh-J6c55% z?@&f^xg#C&aqlm}@Ajl9TOIb@_y-0})tZWY%vQ5$dDq+=9`$)rB7xRW-u*UCgPEi0 zJ2?%jF!Uoll4cAA&sO9~rWL)T#Cj=J8LkI`Qx%iDChDVsV6mfk({962vhz5PoV)u1 z)x^e8@ZcLb^Kn!T;UoPN_D9W9{u*l|7Hdq(!fqiqC2x7m@2*HG9@>Zv_A4_uUkyrc z&&X-sWZpzjQi&Oa1$OB9cQ2GY7bo+97oxJez+Qc0&dN!B-i}x>>y8-G5xC^>EgkDk z));PO{LdXXQ%kFI^KjIYcA)JSad|#uY{>F!)z@Mkjwt9>+@WwuC~`UzqZt<9#WHyg zVoD}Qc}H~b-8Q=0*&V#Qk$tdZLx_DB*638Nn@@+1PpgrX448@SzRqkiaZIUKhF`5= z90U<zx->M&xToujgETSRCb;jpl8M`+c-7WyM3?PIOuhmY65 ziwA22c66J`cK_(QiE)tj8p4d^XZkBSojtTUS@&y`*pgcltis_5&wMQCQDg(0aR-4ATHhnDHzHJ?Sg%_Ry2Dh~qSh3Qtw$EIDouWW*`~4$P z?g{8h>N-#MD;=26*rw9ImeZQND}iywv6C+O({<&s1jrc?UC7dWexc}FxVoukkO!+*f^XNCTyeJ*U@8E}a+p}++#Q*GfqshxF zH$Ru}E!{hui{$yVJp=daCda%^ESmAos2^Nz`En=v{^Z>0qDQHCH|lA~;}xyPA`$q)6& z2*R=y(UMO!>*7Cm+E!<&IhKSOl&EOUA3=eR-Wpcu8i7|g^D8QN8kc@75SuV-iSh3B z7=R=B09{9jFK%Y|eQIg~a(Povp2vxS#wYY3`;i>JxkR6tn(gK#N7K{Nv6HK$6eim~Pjso65f>?e^01`u7Av*ekUN6 z5s#rG;^1Y(%WvoewA|2D|JR=RGQ*C~unJIHL+XT46upe%tzm7)4PyL|EE`oeO8E1^ zr0FydH)Tp~)y6S+$ZG*cHuoEW*CnQAlLJ>KOVGy81S!0R+j(n4@+YP38{K2@u3`qg z!@PF;I}^)Stl4>U?g*Z$QPTUbN?R8R!2q;fBx)2 z2d=Xfn>f9j4a0nn-<#Mq?{g{IP?4rjm2%X=UhdI{wVrdl)-D~RYBwHY)~{UeC_ZVr zVgC+UqIwhB_uh48xdFe7jsO{Jo%d`{Y+ZitmwY9Qr=&XkD`;uzNAn?oY;^VDVaKsi zL)l&Whcn7nJx_0gmseP|*rbgj*1gDg(y7Ew`luP?n=^d&XGxZke_!QR zqKSjzM%V@J?ulKDti7n*G7=i3uU~Pjf+P>*de2RAsxQeXn|wgpDfK=X-A{AZJMshl zzZL^jzQQ7sss<~3pkgnyu3o!Cpqv#L5I&vf{;9z zmn;c6LG!9O;+<&AVIz@ZESvTZLn@eOdAcFN&16qa-2L%wa@yLDo%OV5Q!z>7N7vlO zIZ|80qHezsCl$4{rv@w+(g&A3KG!mJG(so&g`Y?MTtx**&>-M|2Q0}Sxo1tjxVq|B zR1Nu|L-d&?#6ur3`evnwSe5+XvS{t%8d1Q>REOyi;r>$dee57#kO2h|rZ1L!mlO)R zE1t@NXS0cAH^S@y}QOt}3%3kLZ4U@8XhIHJ@OJ5kKf`=hPKZq<}FF%&ER0 zp<=eA#eQXkErNk8^ipr(;gA#=sb6iM0`r5;Ym-%0;k^MWRatpA60lsZG_~Mz_5iD< z0Li@P@?S%a%Te$t`UMtYsdt_kmOS{)r`c*4dB<=B3D zR%vG*e|Hy(&aEe{zRz_6WxDpNePGAd%ShQrajRinSyV>ubZqdI(^p0u(mPX{M~jEK zEAjPIxG$-x=e+j8V5A5WvtcEQdP6vn<3uXdAz5V^gWyaU5mHg(CmyQd?Hkw=@e&uE z!0fga6)ULgPhzh3HsbGt+L)qs&f}I`r=4|J*8%#PIBRE2E{XNuM zG3?zC*$W9HO7N5vs*=EyKsEsFos0+{wzYc#D^h)_z&Tx!$Du-e|0MZyjBupR@6C;P zBUpO^eZJkKhuk7(!&;FCtEbOSue!&iZcSz{ba?Gl9E(eY0~oq_X%GSFkdO}P?i3Lu1*BV~yOB=m*wT%3 z?{97WopZh4J?CG29(~_;uQlhGV~#PWWN?^hI^xFpetS!XD&5%~X(Olsnf4YP28kRk zBAETN%Iqrp(6Hq?7@yO)$R8$?w(FgY2mLi9;B4M(x}((Ku*h9~HJsRU_W-=Ys8(!Q zA$Wy?XB6eD+g8^NjTj^jXU(dh9BrFh&4(AuUsTS{UBtj@JPbdd-XNV6!Dbr+$n%M{?u6iDs_-yEfsbl17JPLy z+4gZAyb4M5P-!&DGKm~D2tGpT0bNLv8&6xe&hBt?bS^lLXN?tTi_|?^wmTOfCiV7p z*XpZDr$r9Yea#II4-jR-eO*-bm3+2opw(9)Fkx+fX4$?12@IysH8Km8Lpqos9b-RyjY{^_a1Ez#%z&1(bV;yxkbU(V8@7BNwu^bDtGeWj{@*_C?6ea{R>l%7Z7eOV*Y$%T8>RtYeiy~b zM{>q7J;dv`jIk1X#0ap~EH1D^=rb;~UqIWckG18bw5Y9=Mtqsju>4Z7T7qf6ndTe8 zwBHb&y-VF1|IQ^BUVW1|s%}Nbtiuh`L}INm*+acg81V^hOv^?=mjeDzgH=stXUs?~ zoU*~CRHO>}Fm(Zk|4=RmX!fy7i`+cvUvTN~3gyh%dA@IWZ7)b`5verf{Ou4i3f8^D zGYF>>44SRBiWRDc`Dupq*1ww;dPyqiP?qre!cHI!_Et9dRM<4KQ~Yhy=mj$Up@au+ zZ0_)O#8kZ)-t1(GR7KSlx%PtBv;LZg8Ow%28%UD)N-7B>V)X~|Cz2Rt zyJyDv(ovvD*>xwCLAftv{%y}6B-QB0aB6|xr5%ax9riy2g?ohwn!)qUr{)eWdWUJ; zmEs=|p)vU$_%WOrvd%mq1sV^@tBi@A1Wzw}3(F0Ad-Sfjax>%`~5STcHW*;t*+JVV40mg&rgJK; zDMEU#cXQ9Ipi*O(xfXkyx|xar?MbbvzBD!Xkq^}#W|0S1rFDHQtx(pS9^I{d7LOgb z{>Y%~rvV1W>DO(+M7pgzannA2y`peR3RKO^6h~G~!LOjvP_835%LAfs+ak*xB7>6{ zK#(G9a6%zu4hJh>g$tDWguxtid&Rk^ZDBWhKaT|zPtTitA-PiYaTOBg=s@PeSX9*R zRO%g0(s*)MVn>HKbv&Mm@$1NWWFusbCLOY}+5ZqL3K)mj^vtpfzFd*NvJWJ>yZ&(L z&@z~Taju*Fh6gfE5s&i@UQ%i9_&wpAc5Fl;#Kugv^>T(Hb)ZFC!RA3I110!>;2-R4 zM49ksxZMwsHZN|_Ju`#8N-&NFV($v;h6*oS=d}NN(~iCzEfmXelUkwlU7$*&~H`SCFESOFg;W6Z`>lj^m# zsvSN8>!m5#w^dNbTa-di-LblH+x?F?+PAmQv?}*|nc9vPt-J5`vAc})0Dw#Wg_}u6 zfhr_64xM)0mWkgh_Kt#=ZtnW4ek5Eg=8jgjTrRg6iT+x4Vo*1yh!|!%0PdKK$CarI z`!!~GX9N#laIBM;b4JddYxJJ-qKe;qt&Ne(-t0g8l&7;n**-$|Eh^|Og!E+Dq0yb( zGlLTkHcYjT;z|(tx+gO(d!jjc)ghatL1kQ;biAI3D+|pYK2yol{-kn5%+d-?cl=A- z#?v7CF6@Ib(}t~K-o70n5aLjp^0e}cT$9^u7vG@p3`n<9=Nzv9+Fx)3iZxyDyqOXNn`8wgI(df zWqse8JMYoA@-TdG-8k`riXm2b*&*aIjwjcm?4pm=n1;P^r54~%F0*&cgrjZ3x`x+C zrXf0#_$VJH%-}rCBVJ2oHBSUT{qjv}l~C+o1gdcC21h>h`sUTJB4Nz`mM6w^-bW@a zZ;v?gBR3(-NaP&*fr@|a=iLJeLBfp9BPd1ksF&Bt2#vCXm2U7AS@O0%SEZz88Xkse zp%LQK1W9I1I8zc3+0^^{r3$egygaZVF(ewQR$8H_e~+I zTW*R+r^L%D&0W}Fy&I?#Vnu`s#(o02Xz#O@p8wAKbU1IaGnzhZVP13mq?r&9U;kk1 z-^7<1z}B*Vr)slFos}1xf#9|v_cS8Y#uw7YvltA2gpiY=`d3k^)PnK$y%u;<2ukNc zZNl8;+Z32SCD&@f{QEu9j2;lG0|8oun!T?Cg3fi=`_`gX6SrgKA`NxUc(=AEYVble zM2F2;j`J<7Pfd+TK)-KNOYrLvE6VfrW)IR;wdLL1Dh75sGWex&l45N(mU`Q*NfmbI zj(~wwn*yhH1Y_P@I6&7R17&7H$Fpq^KG)^r&)UCkuS%Kx{Qk<&wSVpiwfzFrZ?*@E zKJZJg6Uc*+KPd>zsgz#Nzu)LIpc`0Dym={)Cb#;`H4tyJ4BbGHcV%Dp)Z3=Q;)p6S ziU7~+&3f#a;=D({VFmJ&yiAWY^q=aJFHl6sO;#n zcIt?j{ytC7BmPb5*ZdQpT8CdEhv(y{)M7q^DFx(kY;gJ?sGx+Ci^HsiA|ijd6GVAg zVU5$-ol-hCJLY4nTO@N{eR5IGx303lJJ2v1Q)S^2OqtHQ1qeD*Z(Ordf7a#kRC77P zL!-06wlZ_^{w8)*hzyFv!vt=~3c+#L!xYx}h~LP$NY=7SzHzIZ`Ss*k4l9b;-5_de+*Y8Sav7Cox^ZJIzC4s}wK3deY{UTSZ#|K6rkuQ@3zVMU!MpBAxd{I_T$P%VIfF;$dmY|_ri@%T(cbJ1 zll3Tnl+|9|tS3(DGLX6y^NXNXSS&b%U18FyCC%yG)Wr?cH{L2I_YxZ)Odb-P_7A8} zmE2iUc%|sCxRZ{13a5PGTno56RTmK;U~8b!f&N@X2mrz5M@v~-*N@xR|8q%q8T}Kd zG}E|GTf>?wVbS0Bh{hg#EHrKIaSP6$1?yrRp%ug$yWIIGi(>XX?9Zt;Vz>xvdBCH& zl@w2ecv4`Nc{<_V+di33+g{D;_*NKfI5Po(F6Y{hVubD}g3xQ}Fv#(w4mVKUoepDu zf8@fFFqG&QG;S7jWics#+-f1E|7T>hDeg#&DtP-7sgPgYmQ_Z)DzHyGOy4qym+w{o zyEWZ$zUo(%!IxM1eX1_>LQ{)-{~WF9!C}+3)`7sOOKaS36Z0JHd578jXtqGOevX(hTqILKQiQ#t&c=MaMN@IE3_w` zKah(|j+WRr2d#oUo8>FY|HD3CCcH1+|Ll6xr(vzEYSXp!#M@~kL`2Vq=eXVzG>*9ETn49r6y^njsHPY9 z9>h9~^AcC?YbR@m*U#!T%GD}%nM_~1H z1qx8oAker7kY!IrNU+ZyHQ8(LFfi-U~8S5jWd*%Vix4j*OZ zS|bdvmArO+#(vD4gkoot6D1?I0NjKTv!UI z_BekNcMlHb|KrC}ZCY0)D!xrt*FHbbmV0^Y4f?xgcat;P!d~@zo{I8TR%unrqF)#t zjk(bGFUlU1h0c+~uZ&{?og{utx^k$3o@oYc#o_z2XQ^c2wUgO~5-K6{-A`9u+;b%E zVb+R>@*lQ=>Wrqi=oXu|_SgD}VK3Ky9=u-OG4)K9{ zu#o;2>AmoMhyT;8-Aw$)Q+O%~m7F?Wd4_U%puIj`{rj;Rxz)5sB8nH+mX_9}5*0qa9f}M& z>P`WCzr~D1PmLn`yP5^lys8UUcr!R#D_C_^+km@zdd@}1R%jHYAR$gG$}fa#p(1w| z*Z)sLebjrx*h*rdySdr|>mEOH`<}-vAM&A6RL`&r`d8bb*+SuOL)5W8ir&?J75sUM zPzOLUxSPpoXSJVBJSyMM^&p$ATXbi;KT2AWkgY#g)EdKL?7W%YGrbd=!Eb~G__W`B z;3TKvt!Dtpqzhp)l^?-zV!P2=+w;xE?O*thDgNVgeTsCW^s*piwXQqm4^@jQSKRk) zd(;DNB5PFd@ZCaw=&k~~@Hg16q&KdPBaZc_Pgb761N?FxHfU5&m9O1DE$V~(;r!Yg z>@MCz6FCK_pT*CWEYsTaf>!_;dfX(1gQTSM(-s6@e|DYtAJvo@oRgX{v67Bz`%lr- z<|17@Y>Q!0yAnt1l9m1un+r%f(OKNayr0TUdw(oT`5kl`TLv6 zn&oBZ);ujGN-poNu~2&2)LoZD%+i<uv3_MqD96cYg#>!rhM?`=ffDTl7J>F{o6$BK z<_;`BjYx?KUf(af5TgGNp(Oyb`5k8bRdbmIh2u#&f#{7h_`Vf_43P94k+5KKoym-w zB)j%sgN@H;(gUT_Lg3G-6Hga665+4?!UzPFyh5wZQ=RN*LEn$!IE{XBqM*1x2H;wB znSJOVp%nb4XaS*AQEB-FwWL_NT%@i~TW=%dnCY~KC|1C@_TGC2dc{tl;`%ER2i#MH zMHC6xe6u(Tecsu#=)DbO93*BY8&JH-pK+<(Lf^!l=B?(uSMLX(Dy+!;z9L)Lpz%#P z{SGa;xmU{R&6#`QcVZR!lw_Yifcrn&(g3WO=Uy_XWClq5!Es74@p@afXItDUqa44V z0`!&@+x}?Dp6q>i4GPjcpkCq1JS z)is^CtuC!L>}ZjbrxKfb!oWbEc`N@g!SH$&!s*;aOSz~WF#uBfwG||HJt=}@M=K%N zQ+(A3{I%`HC{e(ZtHF6Mc)&juxt!p` z<&HZtTU&_u1Oz9Vg!x1nS!*ulrH`wApk`&1t(_$>7Tab$VfhG@2yf?O+tYf z@Ere(cwO4ntgc@L&$qL2a#nb+j_t=#W2}GX!Lm`cWbslP_KMc;3#to8k1wd$x^p-x zls>4=@75CD;WSjCTw_&yV%6Y&BbvAGFo}HoAA z^@tbwD)w`dM)va5&_iJJt_X@=<+YHj6IGd?6T&5i$_XNva8Op`9)uz(VW2_G@A)aP zVVO8V#QVI)yk9phNb-Be3o@$ME3+PwflKgPS+xiRQ_EB|U9~k7`*|1=5T8GX9?T6v zl~`)zIZuiy32K4HV=u@ZAAGw4P|PP(+i#32gbOMK@I^!g{3GTuc2go_Jmx-;O8cR?(?>$ZRSk#M~F18uHjmb!b{zC>ZBfuj;8 zJ#vs__CdKN4E+1!E`QUtBdEm_PXte+S&C*pfWGe0l!n}w!D;py1#RE!q1%wIvEX_U zStx~Hx#Dh>Wo+5zj$Mie>~&(&w$LueyH#%}=&|Qc)-glatN{ygg)XS2Vlq(@^Ryh4 zVTL+VMXKG9aqZt1K9ro9si=Le^sAkr9vAzG<=hg~01mspXJx%)ydQkffvwvM;^0)H zggMro`xw6{t_!Dwm_o4Sf>icazuLtvA9qp`JiZJ6UH&3-hC8`mYx#X7+}3)Nf98z_ zqcfsfsLm+IeDqou69Rc)Y#|;Ht&I&L1MS&iFi;r>1wG>>qLzj%c*$SOqTT1dVw12v zraAgtDu%qjVJxjsd#Be&j_eyTcOd3CB@0sDv%94I1Wp%1Gu_Nkxj&6mg zJrd$pUY9SVQO<=t<)y1YdtQ!GaZu@r9deaNY9NE~kW`r?>`F&RiW7cAwgO(tWwR2! zzj3t>svDg}%1EF`P|2rcv#!J?Bc=u8nbt_X-{y~oV+Io-LNa3VDMueYmem7dCmt@K zn5^kw1e$4aqNJwJ5Z|)3DWO~<46G&(O7Je{9oXGvi`RJ1nxowDI{PniA_knC*>I~S z5+Hh>Xx#C5D*IPN_N98K%6G^gwPp;$&f?=6*DQ`GB5&>)DDA!rbwJaeuuR zN|hJb%{v>W&#xsUs;R0Jdj++!upCaRP&{EE^;w*$pPfd(qKA{ATJ@7Z)g_eZEh<%P70HLWZ}d z`8~Xl-~e4X?3ENVwNVJJOct{G!Njw@YKO$9ju zCl84MD(+e|LG6>XCS-ne#=!Wlp#8E~ycP=qKgI3wc0!Q1_-5y z5ZK9`b_esUwel>YeP>&x+-TtGj&3@h|HAlY?L@Z1%abicE1mezzJwh8wjH_{7PlsG zyZUTOn49}%!;Qn1pe+yB?FSR{OGdm775Se zfwS42>XZvDWB?$2|mMNeTh0?+;R@iIhQ!RXx1pVA%5X~^-W!SYU+tx z!)-e5HqZW8FlYm?q3Pa`N-H)G!R3PbDQJMQSrABh+s=O&N zqZId=I;n1%UyqbDC2IR&lSKm8OBAfFq;O#-%C#2rx0TOnnsZ|Efn3MKD+kIE*pA;U z2{c^H#FD$*hSXsiFe<;*1e={nlea(pi0)zG zot%4wmls2l+XPVP7}h>l7T}{X#QxNJ-HVfzxE^X5G0$PR=u|G0ejdzNg?4M=EcT7Vr}0Hq{s;)UrQuIA%on%`R=P zVxOmvhvHSy07xAYGM7|cO6ZV~Yj`;UDYt)&CwMl zQZ2oQNz?3bg?(Em=!zH~AU((s4-yGYG$+h8g3{K>y}r{>#!1y6*yI_@y%()kvCPr;v zONok#YHlrB?FjMk5P^@Vo0=N*hE7*mQYa*!ZJ#FklmFewjA`qQ&E_=>;VAslpRZ9d%F&^97xSRXOE<>4y`(`0&?;fGC zd34ZuI%bz|Fn+!D6m6g<KYb#wWy^z()z zQ*!bmID7uifd&IiVg1(Tg(H{Ccl_Agt!Q%YZ=Y;!Rq3j{l98<=OJ*I^U)#%T#RTN} z`o27w@x|?I4Y#Wuxy=_h?TJfdlTqU%ab%>_;9qH_cWjc)e`yietUkiY%!$e+nGd>N z#?UgXe`$}|xwd>&R8$T&vZ4)t{p{RDg*3j7Ce`yGP!zx6#l*t0>!xyn8_lUotj*0^ z4p;jOILTr@ehdOL2^qi#ZK>7roz!u_8B7xm2OB`>czEvG*Uptp-X(~2zw@FNv;r|n z#yI+}$Jecb^DFPzb6jyy?25Z$GoN||fd(~M%~CY~%}+LGyf)ARMezA-Fjf@ij{v?C zKCcIwBk?up#KxZSCX8pFLV<#8W)>w5(Md}iU+iu9{1w)-PpGvnv~YL>KTt{b zs?nvy&FIit8wh!_g^=jt@XLv1_jFHw4V){O;C!i>lTL>i0Dow|PA6{C0_+gfs?36bkqhLgb5|WIX!84xq z*&Yv*`RrxoRRLs(7zE_1-5SY&R6&lC-AF<=e_+mU z4sRcDWRyav?fY&+bT{hM=a_+d;)rrzLOwZ>W}4062D*Hnc&yO*NXg+C9lP$5OYVyb zg2eM_rsOl)qGoW_WgYzDTox@zK7l#zirJ-zh_7V= zZp33IWKMf~N`4P#q~xZat&>Kjn%bHuL`>1PckEB( z(&)Cg_nMA2*ci^xU07}_@9L8I@bM%4!-rk$zvIOF-gu0s$^HfHqyyO1iOO;1h{|y` zj&kY9>*4`E0e2Xu4@r{3qB1LgX8I*aBZ2A%KS_)xrnj!?DF=4ri4#$&H+ejIv#1_f z5dwl-{6a{gEgM?D_f*Nv$cbfgG7b3FCuh^2Ss?x; z*pK@7wJXstI_QNsD;sUeh(Gv&Y zilQdRuMZ^+!G!6Av(T`xuXSruzKT013HAm;+u1saVG7b!Z>)%_6|qjN3j#J<47Rhg zSa6GdEvDk5wt0EUM_69_*A}D7SU*NFVlftzjJXPuDo{d3*xK%7qxkX3=gnEG_o2=x{T`t3?h8$7|uI^)!Rzs{U;R1uL z!+H3^(NTF0dmrfN6+5uC#`wq&MzGvL8MI?BUh&;8`t_7Qx?UQhjd-^1-qXE6^2`#H zp1TD2Ng)Oq;WM+ZaB%W0dY`C|F{$yVNtuj2RpVRB%r91P^afg{JR2rS=S@Y8UEkMs zAqR>w0p8qgH-k6WM+$x_dt%xse=gGaV!6Zy?Zm{y%Wvic5i$&5s(+AK+Pj?g$82nD zyWY1ahgqcc=PyAnpDzL6`D|6xoELlp&gyHG7)!Rth(-`pB1BzprVeUjW||&)Ldu!$ z(x=9ldF`pQxuQ6C@Y>d!b^R8-Y7hhCbxD=cgV1QlhbUyjdW9+bSV}v;YBqP#kjl3y zsLDt%?@p6CfgCgETb)k$d{3cUvy>*Vlhtd8C|}<1!C6j)pqw-U0*46#T2MrHOUy5g zT_K1lGlgO$DK5vg71vZ#U3cjlZNA|uix};Tz6c1Y{tu}`4vqrbF`6AZ+c}%18I3JQ zJOLN;_%2oe?&k>$JFv$+GcOMV&|BxXSNp>K)F=qG*9ou-g@>vMXFT^JcfBCGVgAijxmtQ)7#Z>%j{EWghr`nFP{Z*+e=*2C~+W$O!bZmV(Co%F9fXXBEmN z12S5k{I6y#o}4_AkVxIb-f}dvE%=#09t^tm2425Hgqo`XiNl-;QVVqqN}&9Mf}ylO zj)}ZxL*L=Sd-v{f+AN4rKDI{zrN9(GW`uX|3X4uI!V(l>l9Q6^8rD0zudlr>J}d2< zfZMjvCd0O&L|gQ@3YUu&zmDybS{Oj^T<*r}L$_F7JZ2&}*kHNw!t2Pm2kKN&HbcxE z5swyHSzK5Alu?YhGdz3tG?ss z)AghG)DYS!?u1sHcZ<^MSJ34F&N)MttAcwyurthgy|Oo85rEjNGPHr6-zo`o)tJYHbPTSSL+Jg z>V{k5x1YG~4HYFK(-0@T1lf9swm^;%LAzu?Zi^yuW|1mw7=iT>6-0Ne0sXz*-oY`~apN-+)p%R#CVcu^u;U0_?akoRPTmm z#?w@x@anzS5rD_l;d}-P=h;vu!jEYs-Gz9tfco^*z@8ce^vIZy4NCYHwRAF+iw=FD z@H%w{5@{gygyTI>G<whFe|s^;Kw8nkYzvk!)Y&ws|SKbGIjJPDUKHm88P{+8zu~I!*$dOj&=ABWn_xhQD zCmW=6#v$i*&nU zE8pHXz*{Z!MKMr5vZuv>lugxbFSla@ zFJHQslHi0e{mIJP6kP`cU!^ZeN%wTTE>;EYexiVH>c;DEU`QeGwu4wx2NA(_w{r4e z$sg|(mG|+8?605U98wSC=olEXAW-F<6T^SJig~$FjWzhi#1|UY3|=rCkL6JygE$)| zg_G-I^G?``9!+kzF^It#LE%Gr&z$Hj`7`0xZhd(nZdtY=i7s66YjHrL+*2t~js>2p^<2Cvso0rh$s zf<^TPC{(A+B${DTOv;J}TEA_>c%xwXG?Qu$EvQB?1CeZ+E*^q>@?-Vl&Ye48LEg@A zlj|=-_#QLBg}clM?`)krSqvulf&F>>#$N{~?~%HD+LZq>ndapl^K}YdUQ-tGXL}2FiIgM1 z_iDgHPEL#C>EU>$I*gEoR2MN3hH|$SDObi7>_x65WM2j3^l+>;zu}9m0 zO+brYS8QQnVYG4IV-gcX;2U&CzdRMKYo6M+?)>Awr_)YxwI)vrSA1ESJ$J>0MEJ1A zT)JdZkNeyHxYiu3nUAs9a_9Kmr%a;3Y0@y_dC2aupt9!W^Pmt)V6REtfjl@r?`*7D zjs33C*@}fC*|Nq_QFc3aH&qAciz1Z|1x+0`rU~rky_z0ynK=!D?(J7Dyw#Iw&`nIg z(n=33$U(yJ+r6qi4XEbE7=d?)o>e9}ZR~f7>gU@mSNv}mfpPd(T^eZr*^pM{6M3B| zNln-t?)d0%nblV}Oc21@Qs20WYS8rW9oNV+w?Z@(I%!IhFl*(vRz?6Fpa_l@ifP{2 zt9!Fc%ZoQVXa3rlzg8*G4Y*vcJ$ZmEg#{*rv(S_xx&IyL#AnGA+|>A2(iWEw=87N+ zyLZN`Q+yG+J^0rPajnzs$f!QdycRrG4;K6)PH*;qo)ZgVMC|3|EMz+(AY<|x5g6zs zdPQlZm^JHc9ZNQZ4pq{1F8&@-h-gK5EwE-z4Y z&6wSjLC>k;(A+A;9xF*1e7AI<;!3~(4g101}_oi#;}<2 zW<^NKw}zW$ZB~*G`?naENrrh4bb5;|wwPyE3q@aVlQJqMPHOg3`2=;8MmOzfDhY=c zT~R+OQnz|a=V{;YL8MRHupi4lHK1B}=OzrKiBwS+=A zB@=jR9NcE0`@Fop>+TZOP2mlW>hH$jGw)exy^AlIBp}MT)IAf=eWXj=@1mM<4bOzHq&d`a5L_#7VkJ0t-1Q3 z8=#%`VP$1@GV)NkX>u}|5*R%Ic$2(%@;v_Igc+oe#c7Qpp_$62KYgLRdE`}V*i=DOAbHAwdQ^DnnXdp~mUTm&KQw_qg-)re%fV*5cA%hd6 z<3VS)Rd<1}U6?Yi_dYybaJ~J3PZ|;777i>>%lP0fT%?#8{_9U__?IylzwV_AsuZxf0#!FM!g$&ghUdUE?&R}{CX9Rh zR0_G#EmJCZz*+{mU~C`t`1 z?CJT@`%g-}Cg9xH^8EIn_gqAv@rZf8;Q%HZAtk=hiH9bGDyk)IJ7hY zv$VM3UWM%aC1~^y$yvij**IDR*g_diy?xlhkNNOeGUBbwSWuaSW-0aBZl~uO4%Ew* zZ3oZcc29by_+DC2l{A{M8h>r7us|Y+WdY6F7_!I2p|PsPGs&+CIg;KaPBiJfd9Ixb z@G(Z-TV`n|Rk{~Mu;GQ@J6jTlGi=ESIEm~(si*W29DNT;_ipcI(E7J%TxY>e>g znS+q%(0FEGzc2zx9CQv_DwK$%;tn7Yfu*3}v$+{=a>`DDT|3u9F_LD_Gxr@aP$B-h zYsSKPwF!fMB6hIbju=VNm6VpIe##uL|6SLJfO5~DZ=ZVC)?+cqgLiz(wl7Qdv&sHrQ}1-C%#Ej&ZW@A;Wxv2cip@U9?R# z>q(N{)PQHI+PyyS^9BEU?K}Aa){4F3UKsoWsM>=V{H^ZmX&baQU5jyYO~S^?9&{`7 zkqB;|{z*cAV7ne|hJ(lh$hi04;)J>>;IxD!*Hf5!Vl~kJ6c&}mXEkROL};D*aT58( zp3%ck-@-Zo*hJ@q@o)WQjash7C8!A3KabR-@r*b2SFF~YrSkQ#6mA~70+MZ^=E$P| z*>l^&o*vyl7kDp&vygEA)|&K{Ha1mMpv=Y3Ju_1})|nwxhOFlX!1!}?ObLpsOOk=w z>7ZBJZ5gd!9OTmNKVcHwLI^-=5}V}|Vnoeo@~4ZpA3DA58sm}n#QV#R=jcm|A$YSp zX3i>v87dzHOwhEF-K#Vle82~1B60tV1r5eShHJDa>9!M&`pGvEi z{H?`Z-VNHAMtAEa4EQCy@7q5X`18%L7{v=aXBBONtLm16zl+^6U$5nplkt9VKd7Ss zSGS=li2;}jbdRRPL9)8iDj5JendW00H%U5G25!JOBddaDpNO6y_9+!c{+In5F@^Qx z3GCQ)!~gN52OzY3v6LA2<&-CO$nZy3L1)#kj-jB3Q0pRb^$uKISr`{y(B)U^dCfgi zr2P=-a~v{XUo2|p@dMp3@~v7h`{v;4%v)MPb$zi-A?i-MwhM>MGo64qS(%a!5YyYS zYZHQAQ7w46q4$Pu2~(=(8P}YQFv+FjZ~Mh6Yr-vLm)cQ4kz|n+E<}XO^6inqSI3m6 zZ}FIG@#v9*Q#r_T0NWV?Wvc&f#P|yN(0hDDtlg>JT2UvxVrXB(331 zyKJ4HX7<#;`ZakUwO&2PE95`X^Jc?I22c*{D^#T$w(|7x`kRf*oEN>)AodHjgBR?k zOf{-XRGL={ImCJ%4?x3o3|2+wMvT}5rnodJu1)$Icnj^b3olkMHJCB+w+$9!bNiW+ z(}pl_s%R8HVALuIK(D=^h4CCmUiSR_ff9Om=IuE9iq|1N>*d<(le;dR{vnOmujMYM zxU>3Wb^$uP()RA+(X;klFyA^(*<^woboCYzkayf9T?+kqdeKi2rXIg8C4q}RZ)_Qu zmGo~F7%rg26o1^Wb!Js$%F{76e6wYs7g!J^0xa~9%VQO^6qb8vIke1-s9ft_xoscaS+bPHU3kOI< z3%;l14dDNF+gE!-y!XzX&z@I3l^XB#q^NxCs;Z%(ID z(+F5{euOzAtzdovCyVsY3>~-bKlm36cG9T8Elk3s#UKej-ps;$n{Ew4C?_W|U{7#2 zquqkY4_WXb5oEzO%F*y_%{cN)%Hn!rQh-9;WDdKUom`?Xn=yp)ZQSCnP znepZ|5(6xR2(CYv?R09wy9s`qaFt>wXj#rCFN)yWk)pU0S?1sne?(KB&qg7xot(FE zB>5pFPMv*2*B##t@cpCWMBtBkNvH!{q($%v7Aul5k+37xhbL2ZZS#sth=Avy zCFX%5yUpf)D?wVH|l8^|g4wnJbB=@B0G#U7f?;Ap7UR6@a z3#pkaf*!T!+aJ)22hS~&heGx$p6W&Bgi!&5kio>>KJj#u{AQ&b#gLzAX$~5BKq@dHhYWF^11gw$Iobcz#Gq%MW zt}uZnLnj!y7dqzcMl_85b0*wc!-6}aB#`fNXF26m5YbRKr5lPGi}A945> zMemwSr*r+rm1)cJZ*gwV`f=e_C_xgUwaEJPPk}v%jo$m23-GjL6s7b053IkEZwDOG zhc^|DI=eOH^2PjZodi-?1o%0ahz zH`y@B-qS40kzN2I3lyxBmwz8bW|C%v~Z*BT4?bH>xM z>%I}#h&NL4tqxE{y}MwvTfaMopMDSO7ki=yFIxIz-7WZdGj)>3#KsP!pkr)XolzcWkMi(D5v|uEV!4H=?Gh772^||1hT09B*_{Z;z8^I$-jTm=5U)*oA+&*m^?R4mQhX*}=r*;GMe85FGF2K8O>HW#t z)u2|znk4Cj4|h`$>5kM*P|CkDD!qOl1e1C%g52JF)?uLqesrcczlMSw(55CNM@c)>^4EW=xmu4qU5u#wAz9gxoxh1JPj9RQ=p1!8xO&9g;NPg`H z)cmLn_gyk$`VJB-%?wRniR^is;qX@CPz_#~dOyTnBVHb;CowP_MIeoM4`^>^=1NMJ zYo~inp4*(KH)_Ms^CXw(%%)etmKsq(YRq2WU4RJY`7iZgS}yy0gbcY_l>${7@IJ@u z5;e0t(us5g#0iB!0Q5UTf6~m%J|JXjyCM8tu!F8L88jWGR-UOqF>?W&G$e$E4^{i5 z)|{Keqd)9cL-sH>^x`u9zVkiA{XrTjw`)vbG-=b=a_?tz?8a3-0C}J*vC~oH_E9pa z)ZUlI$W5X#6PtGV(teA2e5iPOd?326_e(qgzYMEsPX$O^19~P#H;>3c;(GiJ;jP;_ zH{%D7s>gr)*yAwZkMH^srZ>PZqA6j;=XherA|d!!=FyO91q96FWd`%fkKPRT^fn{#uz#2)of7|kW9la z4FsG`Q@fH~*xF(+ZVjJ+cnnqy!N&%n1i3ATPvW&0R^8%1G_v7J+R`pR4Bb5VrCMwL z0sPl+eJxT$7XA`dpaECKKU}^A-z;DY`Bf|O#Mu1q5YNQr4X2?TfIbBn0VUls>=4)< z{*eXeD16^JREAy~Ok+br_%Pveo4vZnxVoU!+qf%|+4Q#~I#j0^*kJ9&-GF-P9SFHy z>|HUI>3;H?X~_vZkU>qItj-%y$G3VYTr$mefjeFe5JO9sl{nW`&(H(>@fa(cS8>)Rg&@=DB7*Ju!CyV6C( zPjqLKMWvk}bOCH4QhT35*;jOtCCMHC^s_h#M*hF(-4$G*GVQtAi8k+M2bHk|awPQdi&g>Xy6ccq+>!W~TIjE^4lN1l|;k6mjbV!?`9sssKk}zgwFP zdJ}-N-=-SPm0AnTYcbFWY~T1ygmQ)Js0EE7Kg8bOtw9V{ES2&&H@@Br8&K+XKYema z6D=0$)SX_QUn*e31G-IaPz5mGWzgu)q}P~wd{$F){u_!1$*{3ayH7KCp#lv)OCPje zP-}tF5tBl0bwp=Pjc8BQbx9F99&}7nSkg4(`~n^pO7asT5X8Xj*RyeTKDg69OJ!^d zn%Mh1Y8J+g1LVy*oL|BE^xv~(+$&!8=e=S7%1-&iEbA+j2ux;!T&jKd#3vWLPRN8_ zv>C)GrnFa!8U%?zbSuzcXE%%>1=91o-Y1E2Lu=18gJd800$o>D6QP-K0O5>wN7)-B%70PTJ@ezMI}*3+(JgZ`zaU%)A+^`k*fl>T<>rhz5KVhdl1|V> zeqG3TR9(52?#B{Qg{6AeC-+6xA^dq6flA&=Xk+t!$L7u!&@^)2(GU?NSmvPl2@aR& z?|ew`IDFbr3J?~(+d~7qtU&hv&XXU&8Q{kUzYG`St2c z`OwG0jEQ(8g!?fP*$4iu<4EE9Z^idBSgMW8$Fb{Aw87lEb#JoC+eIg9Cw$r#*^@rf z)cw>OQ9}BMv)SMf{+q$y3oHfIu^-uwkuZ`N(^ zrP}&`w7qp$R_oI?e1q7e2!b>!snSSD2~r}cq)3Xibhk<>UD6;8($ZbhDAL^^-6{Fa z!u^Zqd*1iYcO3^C-NL%px?<*>bLKkd70>;)KK!^-=kq%LQUzOS^Dq8cbZvUf)A=qd z^&5N*G30SBTjC&w(lD8S^DG#FHK+0e5?|k_#tmE$FavqZR|H1OEw6|NG$XpwYOe1) zJRUfDtK;$TNk%*#;*#?6#}f+f;}h0mfUXdzi(+0O^Jj4l)0(L<{1TA9+@~wAg%*31 za@fJR{4(Ute?_$;p@V5+oy8qr7T0ZJ$Q3V$yq?d>sWs6kv$(j+*dE|OpJVd9opxME zHGk!C9rrE#jlnfVE)BUJvcFcqBwq=`@pSAtNKDD4HqojEzDwTobCqf&UaxF8i}(V| zS)qKXH^7tCsHiuyelg_{Xf{$gK~+x24NO@)va`fOTj+tEyOkXl+e|44(2NaW6xMab zs}FsQ=g{R(m{F*JDKH ztUgco&|?>E%wVSakJmX-kx>LFYI+=ak{L=*9q#!hCmQY{_57E zS1VtftTi=)QNAtQ69&v>Y@=RRV#1^xFw91>co2=S=?GX2KrOAe4jH_81eH1Of5|HZ z(CYQQq1v0i*r}j6w6O83Q<2JfAwM9!0wO$}X0pd)8#YSz8)}rH;?E9NwMIODAz1>j zTqo1}vHAPkvrf#!ygzM;0*F40Kjb(EC*5!Ov{yMRhr;pTH+8R94_@*c_i2pmd$uii zsRL^-CCV0i`a|au;SzsNrNiA6HIAP9$GAJ%j)c$EHJKNlW%CWHu~+22(Kp&629UN! z@R51^w%g$yE%6>DkvJw&d-`*veYBm)T;;#JY^r{Xf>X1eBMJM2hvVdl)N<|iWMelO z`kh=-eRq9#(}+$xUw5wmm~+^2|i5WhC% z5qjnN>bP}6VS~N4`_z$g#cs3c$nmz6Tf=_2p4(K`>Fsk-RqY={(0wBKFKOvwiP~lg zBwf2)P{`Axz&`!m8S9NXuSKeXn>8TP|+Q4y3`e8N{4oJjeIzwHm{>HcH>>Zlgg3GnbPjVO`1>9L96gKc zyWPCc6y(My!ynzL&+?ZRf@3E)d@}d=U91E^dD^IdR@fmS9}|H68xnqRugt=NUK^#9 z0yvG)*eB_^P5T}Llv6XWcy@hE`=`lv40{eGXcd_CIX$ z7wh_wBe*a(VDbwTG@Je{Dc`udM@~xaLm(@v@n}rVF97nYJu}hP!RL)fI|rSef)ne&R2iE3J-3nXNYdHMD>douiJVXqj8Hg&=`kuh>2L^OK zSB%p~U!C!t&rWq%BasXuw1jw`+N8_$r1rxr(X!)Zo5WflvCq*_$EO`8pM{fG|5y~p zXA>|z%>9{ilKqnVCb&@acOWG0Fp=*i9zC_Gf)s$|mzFpcX+jZ@rH3Em`2klbjtO_E zW6Hu8VPP zO$Pbno}O1c$ENKgedrON40rZ8Em#8W>y~8cx2(O^qHj zq)Vcu;@c8A!Q z)rU^vvz3RR0hW%n6J9OSC7g0y5i_MzchzwsaqK;V)-d535cj=#q3^i_60&bwo11Pp zu;%WFpxxzj>>vXDoTb#H>7kG5mv(d~OKHyxKUJB=nunv~R-fjj&wd@5a3rtfI4^yD z)2!nqno*YMk^Z0}>Rp&ZPHuX*e%_^D9D@p!a(47zp2aS?B76;`jKxV`A&KCEvMz6( zn>sP47T7~lKd_18Al!H(XbxU0T>T*khaRI}+adoiOOaaj9VqaVuLAM>x(*OIP*1TS zWF8qF4vJ2Bi?r3o#SQp;JU*>E=)p9DBeVcY<$E4XW;oHVH^gFIm+RS`zv(XZsdPc2 zMALvrC#6yN{*sbglEpu7rPNn@&2hVqoM^b6%ZO#aBhG_5H`!u^@fE zzMi2-d7tXL{_!^`jUbPEEHX84OBht4vm5L`b|FZKJ=(1{DZl?&R`Xe$0o6jtAtu~v zs21Ai&87Wsdvb8I_O;MLQ8B`Z2-0?3NN**%+!iNh+IpAN9**4~=m;_hZx;+R&wmv& zTJp5qLt)2MNWWLi1?|qdLCp}ib%u}(~a%rw&3wCfo2Wo^udN>75 z*r^`YT*O0jURU6@CME;Pj^uO^)nM%S6MpfJ-llpkJr7Escr-gj|H!I6jU9de+ruw^ z5gm@|?{%}`aWMb-bFSPCCwANyl)}SHZHZw2mNoU8;XI=5Kbt}^oo0wjhAmSn^$^

hQ~8@Se$6IBtkO}B>01#C`T(ErAmie63Cl7$i%GdmPs11 zom6THAYYA*O;aA}3ne;7ru%m3(d_1nKXE)Q)`g>5vg=9D5Dgpf_&O;?#VZiFHc8?iL~F&eL?EZaQ1J9u zAu-W)iRpEP$iqR$rGL0uGOqTH=-&XzkGIkdm-|D*8^O-K zj(GF&uTNr^$k;GjimTA9#8v#)IxsuWL!4eN6I*+u++S-nP*#u4YUDJX?OYvt-s?^A zyKxWE&EnttktLD9*qxNxUsOBMd=O$zOkQqHV;sfZXCU!h2-lQZ-PI`KyzTPPOY4=x z-a-dl;(5=P`WtWv*j<~+sSzDiA>)L5$UrCosfS(`PL3h3hKi?lrI2OL@u3F=5$k^# z56v89%p9T~(Vx4WJEUmV?juz4$3J+bYt9|hX-|jdle9vyfBk@t$BTO=Ot<}_AENq4 z1;*|>UEASsBm}9A%fze^P)pr@s$GPH&+=OQcOP{)?-hm?v;H#p*fsyf0;)W9DnGCq zae};Zp6&8hjN0Mm0Ws)F>s!PL@tfH?+ifJvEKi@Evc=nh)cLiIetxX#_X}LdKV)-> z%GI>g6+e5k8LB@*zz)M~+5Q_6sF)?D7iPYteW~DHmXZ@Q3F1TZ622MdFlP5nxpVMt zOzrWqSPf{X2x#B_?JwaG)|GcSzN1U9^VC-Bm>*R$ymzK-k183>HzCcS$JMYf)DBS3<4<)0K=}GdOPN)*034T3zJdJmK)zykZ*OIFW z>LJ;C=1r&T!-Ue!{rz>JbP!QVe`oQSCtV7?^wIBM{A-S{dD>I51)3|EgW;f^mvekT zIxc>+NB9JLhP7)t0aOoy1p>yrkCQ;?UCB#*(R>gIAct(r{#4Dml@(_|*n=KeWvkU` z!vx+97(%%Fyd}Yr8 zMJIC(6)w=a>y(UrBSf%YXMW?4<_SMw5={#M5s}^7f#S@d9twn~Ri%sqGpYrJ6{AK%@YV=zBUuP2 ztiJ4be!}6~E=8*1iYukF!wydVpNn|jwjA8J|D^03|5wURt%TUKGm2iMq6F8k%8QeN z1UYm3YS3Jp+}N}84F&2;Zx58QB7c&)Lq-N_P+H%a+PY(T*u42re{j8>hCPm!LW}^yQE^qFP|3o$1r3ycB>oLj#m$Q z4BonCdK_ZeP9BJIk7gsb&v-P9fg&Q+JS3;6c38WJK%4y|cdg)!ljF9xtDauXc#S`9 zqMuSSA?fgf!pKZN0?@%M6yuzPY6|MxAJoOEVDTQx(mOIUTVg^?nrF~Y5(S~RT{d!z zG@g@W=iblbuA-LLHJ5)V)4fp(=U-O)dE4^ZS#z*v&DXkFBHIN~W*{D5Z&@Y6Ahzff zC&*y4`k*E1iMi?T!?|C!2c`%@yT)ufdFv86f4#BO&8Q{QgW=TpAHx^(tI{h+K@(grHU{SN=RLN}#RhXLQ18YgSRS9=@@z8SrNk=@i%(2gA9nd!%PX@csHK|GoA)(H zv|_i(w`6G{!ppb#_{nt$Bt$CX74|hv2W&C1Nlbdj4D^UI$*L0-Fi0cNEq+zO4@^?>H`rBg!5@|uv{~3ZSdwcoDu$tog+GvG>c8% z%d{x7@@Unw+PxvoQ*PE5PMbgT#6)4R4c+|uC5PcdHZ{%yc-+v0cBDcEV?>~?)39`O|E<3^;RL2Z4HgpQ&F3j4N%qE&L`X19xkk;(HZCR z^eN>uk+_Wc44;7HSEC598pu$*3u8qI*U<{ymlx7a^L&)z(U~b`WPu0ul|YBdF}Pw} zBWllLeq~=pCZgD`b8cJoo~4l$>Z+SGw6cQT&+P-Qwx#cM$nq)tm)ddUb!t2H+inF) zIFGK$vWOSNPa&M`bmhug4ffmu!__REKZCAy4~Qg+D?DnFS^^yYQ0)ab-*k6Gp~AQ& zM1!iYj=)tdJgo9*!UY#3`;Lk;q4FzP4e5WyK~#Ft(89umxptAS-ozj zeEH|vD5Bq8&YxBVF-u5YSyaf^2DJ6S!AHF=MIq`yq5HBzS?j{@Y%pKDiq!xKm?+^% z78qNSk7F`S?AKsb$fK5 z_4W(>)Hyq`fzxw5?lQQK#MCtR4uiPPJvz71oUmZKzokjR*S@>^JbF{}%x&L%nBZUUneZV0B9m#~-8wjm zib9!LtgU_H5W2o(!bY-#H*I-}Fu89=pRw|t1^1J@o+94*T2#x79mjwBe4Z5h= z&=aV{r9)I9e}&d}r1FQPx0s{~#y5^or9W{di15V3T=L@fh@=rdHTv<;DKEqi300!; zA(NlQQYe{c=&UojdnfljI-~_b2(9a|QHW;R;R>=8<3LbqfQoT_Yk|qMX!p|XYs5#D z)Of_12tKeuu>7$*Eq~MX)aF4Ik2P0HPH)YC%0;Di z+`HA05qCm`2#DNM*u?Mxm^(G64KtWL^U`mEad#GO9C9@3?Z_}Hqj|e64^2!4EwM^3 zx|nbvCu)k@iF_-*89=e;`<&fa!JRw1ox>K3n7h$Dt^zLK(7b7rG+^^W?eju^^)duH z6;2(m5doQj7+JT=F3$LjH2#7I6MD9c1Zl=^g%zAAu+nSz|lDUk-Iuw66>&z4a6*u zwz>^DRi{b9TG2Hr0qMdWS?UA#T7(sv z+N_%EKV_nB(qJ)eTtBOS8A*8lPN5Z|598LKS=H2&A}V_5L~H%~x~78;71%id ze3$n2ge-FfbW$3$-a0l7HoeBWboD#zukwfFGl@kGP!!7-%CC(&Lhl8`#v5A=qb6r;q747b}KlLb?oHy1VOC{Ri1xcSb6}SVh!(N^{>#mqT-01 zr*y-A_%EpZdG6!AknR6X&!AS!bOj%vzriD(NK)u<3VW zLVg#VczL}gh`Z@`Q9atbIBv-AfMF>)(~vQ8W9U}YdK9g$#xrw02Z#R%i6F-b|0;!xhMG>gRrqQ#T?*{%B=m zM_lcXItEUhes^UH^mL+m%VmEh!&+BE_vZq(gQS)A%on=MaMo*2Xp*R#ir|Uf4f|gjP08>BfNB&9W->ov2YIqaWR97eiYl*qV!1bvv?(zJ7nFm z7ldg-hn!Rpu`1|Nn)sKdHl878gr|o}N>B|Z9Z}JfC#|T#|K#>J1Ny(@fW0z%XP0Zz z_g0Gxkd~YWdG|p4%lhFsHgQ*f=*0dL#oDeE;r)_3L>2R9pkzSSkaU#wS9k0RRFtPr}AV>>bjuOB2Z4Z%w zwTN0!mxiu*+^&OfYHyg3exJ-rHL8M|iU0dULd{2ee^_EESN&H$gP|-U#~)f`KZ_MT zkj9p4!Jd@iz5x#BDNX0ErwLDMutaIR&3t4ij~B@i;T7ZLKe|VDcYKRpD$94SF)$CU zZ1bU;Z|Tj&+*)WnOQi+NXDr(($U)E8sD97WA{W%nfK9P?98HQphXDAyq`a6_2FYnG zh^{C!t^7%S3H_h*^xys{+uXl~w98}>`(N_2REy$K|!&KOD)^Rm7Xi-O8m-i^~K9p1~zEXCud>Y_?BReMK z(^9IB$WTq1bjRPcaP!OM=ditE{_oqXtPf81ygaQM`}&kRRI4Y>`9dH0Wx(3`Dxl)8 z#U$W`RAW=&g`GU+d?ov_NnqRy52x}2bdkCF3Y`4naAh|UgSHjWp0}~9lSO(eQ>O5n9gQNa5=A~ecTvX4$*pb2#rs+5Nyb1| z`*pNw<9L#^U>=LF>^}9c8;btYbwL~T%sTV97yX0MOhAn;`Xd5`bwMT*F~tx5tKo^; ze3x{U4bGxP(W^ptKV`3^C^a1qFXv>yAVYC7uX|&s2jX6hOfNOw`Tmzx9x>gwHxs@5 zTn1Faf6}Cg&l@)gK?mk<7k@(*_#SjEHvZeSxUi_UqBNeu3cnwgm_?@!>)w3XdrcuE36AicEL3Q8e>*$hwA1ELRSSli7#s>E54W^u z4p?!`SsSW;Dh;2h3B7h|7&djtBwG7$0wVb0)*cO7#^9@Gl!S z#6Uw1>Lr5wT@UAy=@N-L2thfy+yzmZ(vXe3lIgtkfEM0UrKVADS7a5=*hBkZK)b`< zQ4Ru>dCo@qKZo7>z6IaEi5I9#LVn|YwF=vL-na@Ziif$F7GgAyXhKyE>d4_VOYLs^ zqw*1N=$qar9~b}}9bAi-|B@vucJ4Sr1L2_K+`oK4fvOqln_7mL^s;0r=GDgA)7|4a zH5*|rp0qjTq#vJ8gIK<=i@$>S!KimKK5#dDyp{Aw&r5pWmAIv!mGUOW=(vSFi z)YWtSSsl#{w=cq=#?l9Z{e5oK@@GL?Ey3J7}m9EUI*lGI6iM9&-PEz`2b1#;$^-3 zsvkcS0XnWC+6TdF_gjxri=^Zl7-sQcq38w=)PGXn{>CQ2B%hPL zCBL@{VtR{w6fP&_VQ0VPcZNSVi54e_++>u77FdLxh;{L<4Gu!OCsj=hnGLvp{^b|! zLI7buqE@uF_sNIW4oe`&Xm2&TQdt3mbaq9%{=FUD*LRno*JS4c3-aMPK*o!I!Y*8h zOqzezF6C|W>z^No<@?WY;&UAD-uE{nqzXK4PI}?EU&D%VXumUm1dGyfdw7X5c(Wf9 znM=5A8plTv*g35yv$DELggmU`BRRT0e(T{oy4PqPd%j+oE~esV{SIw_{{BY7SWvk= z3VDyB+;B`7l5yh>dKef)lT-hO1HrXGqiYF|^0rKDv;gw+4>)6ZZgS}L71t_e)xj`~ z@LOO4^tbp;32`l4fu4|zfj4+4**n?G+!JQy@YDUJTVODB2$2(Uij!h!SPuLc&d1u> zD~yD8$Q8Ixpbp+z)6KBhBmKWtADI6S7*I)GjCGI@=JkcC@ z>sh*-?a80Hz?c|cP^UWr37+?I3=NIGBxe1pHl_Ay@R*f-NF%NP`usp>CA)1p?zn|z zy*7r0+TWk{HygF;FQg!0d5!T%Sa@T-9%s5+Mxfyw&ckvdh?x0=9C*$P**W-cIGJ3qH!|CxJ8Y)rL_ZtJY8lrVDH5;Zb`tl|5n9HdXveBDw zorf6t*+fQ0#yJ$m#f#;y|2?Q8*?}vjwn9JP7wP*Gf z?LL_+DJi+%QNBOW&<`g*|9w3_t>?SMrz9&Ydl={BP8eAi|GJKGbm6loDB?iB)UpiuEF~K@ESIOaZpQxV z;oP?ATV81vXEh>cs2oA&XJ7xc^xUzD7Mic6hDNQYDhdjHvv$eJ$ty0fjZNo4SLHuf zb0snYhzP^UJ6t?IS@-?wq903)xa7le`r$gfrWZU z%%;PorSz|#L%z>6D!~%zS9jq9%e!DT)p?l~=Rv@Q&dw@FMYI@7iO*|&ekcJCHm z$+=W9LnEWdum?VQd*hLic+2qr!C=#Ae@A&~f1`=Q>~N(xF*_T3y2-!k{ke-nVYdq} z8c#_g!2W-I{nIl0nCz;XQ_aS76N|-fDZN5v)VsJiio1kYm6er&Ml&-T$k)ZWfJvm$ z@BD}98G>H^*VjLdr5D!_C9#0mSikmmsS!}!yM&1;A}fmzH^L<+_q#~ph)YaNY|VrX z4(;DppzaiA9e-L?6*+4J=E=!PHaWtX{rf5r4a=3Q=X?)^2FQ}5%$J!F_QH8sQeop@Hu*Z46>xqg8mGLI0WaJnAna`NX6vOwNf zSG$BYft}gtdu5`z)R&c(nJFSDh-&mB^DbYjB1shUFPaPZ7@hs?+qJ{pmHM$thdi4# zH9hv5s2Y5khwtHM%Ek76Em+#Z#O7JAD8s*`^%{V@;{@EW0W90(`qD?0^t?R0i`b-k z9nXx7??X*S(48mi2TM6N4l9%Oo_V$#Q%`m`+SxiZW?Ms5$%~clB65D~y^8Abxw*I> zC@Db&vF-D{dJy#k%6eSj_TC%zn`RR=Xm3Ha0xr6H_Z~x)Bsw}e^Xu=I?y8nuzk1_7 z9H=dwspK>s@$%%)YmzHk)aQxL_7y^HtPSgaQ1Ksdu{SVmIv3fy&Ig}cd*te$tR!WHy) zgfe+36(n%~iB5APhVCJ!{cB*M47J8W`QGWiAS*Q7vIRaG6xsjV$iT`O&G z4Q1s0(U@~8tFAsi%U5JPiaec_eT)f4uLh+8+^yQnIJ*C^BolKt^qC?aMFH3%XsbO)54TpQu$k?41_djBjwqoSxRG+gLw(Bk{qF zQdGicD2iHV<*A{Z@7K=H3Cg`Mc7&dPRSrpzd!oNmY1rpC5Je;*a_u(jYK4o-{^(rP zFrRr2=B7PAxn03ej-OVlb7enE*DOZ+xi3nUN&krCXwx7EF`5_E&Q7lSOT8Ji8yg#( znH+&DJ=v^LRMkXTOib+0pSd{S&$UC_BHZbq;2Gf;Z_xWy;?0|+vF%`*r>qOSo$0cvn4c#> zrs^6IkskBP`ec26+OEh;SAmT?!OH`A=O`T4(Ew9CD>hbrJ6wdNn5|L^AMiZy_2Y+5 z2iHf-tm;cG=FH2|M7Y1gYAmss^Q#rW!NI9qFdiv+5Aui;U_LKlV>f=4&oG^BC0HIU zzj05w;GJ?-BWO#0sy;bjj4~8?L(A;foq3y5C?T$DBG1z=&D?=Q% z_V!bV7G;C?jg4`aQ_Y%pXeb1MXw_0!u&l;4%3}E(rfS}wGc`4>f&KCk~;sc^aj&&QsuYXzjYtm-*85d7`LLmO`R_cM76W3+v+phhg6rsR&lv zqkR@0%lUu>hQ|rfQ;UnA$ocFwPLBQz6dK;RuUh8Dru$Q4XR%jrq=W@E0b8>UaN16a zfXymtnbnd9txPQC?b{fqM_V|^J&~c(toIOq`m`Z{N;p@m$?vX&Oh~sJ1A%g}v2O&M z?rWF>@G^oTA{a$)2NN7|xSfxuHGFZ(A!lmdi$~5X05Kj{{|ux zn*1cpbfy_k`_$rhd#w*K+uK)W37H|?Y#||s$*HMN#KpZ(0kFxRn45o7*axSatk;=< zE(kY?5Kg|``ZAKhEhTg6SE_DDEV>|I21_`tr;w2hV}dUD;lqd9ckVP0KBj3^U{dE5 z`JPGO1}e7r6AcDKum~62a7Z|%P+s}@`6E?M!@pruDH|Ak{duKdGL+$E!3Smzj^5>C zlkqB52?3WA$C&lU-h}j%!$n4HyegwvE zQNnh3FTLNHC>ARJR5n$*(9S*~dbfo(t{HX^mh+95g$ALl9x#z95Bzk_PlCn&G3FvE z`3YDwVcXKf=kMnA<$b{2t6gv~-Eic}>`d_lBx}ZzaGx zNAo!-&>q3UB|uHBuHNtL>?B-qr7Jf(Po=#GOBtpTwLVqJE-fvsJQ*7sOT=q^7o`FI zw*C=VP8KkCPy^xTIB2Q@cmT2Sv^6&q{L{frq@6Qwq+bX;K$ z(`k4N1NR&hsbbavsn;|#oQJKfROMLuqZ@yg{o+MWQm5)T9nA9=FEVBrvn}@LP_%~7 zpF@G~cwY3`xA}U8VvZ=dGb|5(GF}B@El+H+uFHVl#{;9IF==UO9glbVc_ujwdY*0e zKe&+j6jo-6Oq?J1?)uKo=O(sc_f?;jS?Z$>ngR+TnDB)ML_^WY#A1S(%rbRxad8BV zGn85;d;430ThFCo>oXJS=<0gHW=2N^1qH3owBTof(88J)i~#B#Ol-Q zk&{-lawN;l%1V}_jzfeFW^z)J#@^~EC)2s-3cM~S4^fnqlqfYdHDyhJ4`TNAJgE9{ z{kpok048;QRN1&bOu;t8FCQuxUvKp$jb_kr4}Hwy(fF)0;qmiiNdgq9@b=;a29>tt zmRha(Y-{b{;7gNe*a(Tw;vXq*eB4Zcy^h#4I{B~HVgGvqg3ws(N%#8rktJM|O^1|} z6!+#$1LoM@!WWu|W;uutkD^Sf^qJLqzsg%p4O8=VF8z+N857deqx<0M`aDD79wrGV zCafSfA^}#`YyK1h7c_>5KKc7M#qip^eWjOh*Lrn?`!)>?`nxd_$w)(2dM!#V$5+X; ztt+5BS-46@l>c^+wdbq+Rd6RFVDdW)4HW^%$=P+p^G8-N%70DT^JG9ht8%m}DJwH* zptUCu5fjsb-4HI|!VgvwmSZ-)Sa&M8WZdi5lO9lj+trp2j)&%AFzWeWzcgEd?htVp zTt`hnOe?=CyJRulh*e-bdQ-bKL}>7vc4vv1mQ*-%?Ys@qnBtdy0@_RTvXQ}BFrdDT z-EQL`_TJVe67ov5%#L_Spx`up7Vf3Lv-lZf+O@gM;z#JEGI%T&5^7 zfAR)jk_Si}1H?fhnAJ;ISPif&c2Zb&szku+q5P;?M>$Q#d}?Y0F)mz45PXlmzO_{k zI^^$_3k@_OyqoV#^pw77xb|aXjJ#6Xc&sAbhl+xNVx}XG;`m@k81C`ABhDX0|A?Nt ztc~#@;2QxVU=#CnJIJM?x$poV)qx4#N$K>V3YByRt>|znw{u=(kguE|X^(tx~fcnJbWITcKB9Waf(r`RV^}aM_T$6K%gbvm zv9YoBhKue2pz&nYZv6u52f-F#<2=FfQ^S6uaN5QLr^I49A(&MV$uhaPXt3RM9LWNR z-eayKj?q7twl#}ONIatJVdBGl8*IHg4%p*CF#Su8f^;vx-3Gkrj?hz9OMMLx8nV0{ zU<0lTuO= zq+@vODSXp5U?vtFb+2G!i^6aX6`S~j3!r9TxT5a-=MEv0S|)-2c*ML_SDBTeHb)mo zxSGKpYmen4K|Ty(v<;Ah{{o9!SwEBP9N0J>`sWF5tRCb7&OS^1In#hAYZ1~5A~{k4 zXHuXYeo5>tq{c1AjASSmG92u#c*FcrLZk+9ttc4W=U)WT?)a5z(P`aG81eBZN|w@B zc&0bjqUlBRFkk%^UZtPUF|cPJueCB6)Gi9?p+-D$2oPOyeG z16HkLW8|=lWE)!nkQLT(d(2xELI(-Ug|71ua?Oub^1}AXc@UefMS15A^1o1b%dv?i zWx%GCn4&$6WEXn-_U>2396yk|uLt+7;i%Q00O*z8UtXo$`W}QS3S8A}olD<)d*$sO zz6ML&nW4x8%Bh%6hr8|o!Jq&*!IX9zY*qW?!OdeR{U0DtdMx z>=&KZ|7?ADc&KX^;ibm$G>TLH?q-Zy_!9L{>pVh0K#ZQRnllFzEtn}8Cs74!+wCcgp!+^7>Qm0fMboZx5f!nnBT7*0JU!VHKrxYP zmsl++WHJA>V6*3MiTVfzQa?taT#k3b1{X*;43Zql*|%fFvlNz85?Q-^s00KA^7mvy z@R8SKP_KH3nwVS#Gi?cvu?UIp4mcd~wmP2P-T`5Y4B2J4KZefsFn z7K*EpZmrV?Z=yjf6SLDN5ZtUjUt>RD1UvWN$~&7@U*EI@wU)C<5extP`4b$i9ARis zvps-^u1Hdqyw~oX>LKTtva#OjYoUG}SLh(`3|ud|?@f*uG#71_2he0>WHPomW_g42 zSgsQje}sub{JO#B-kzd!nk~dsSlxAo56hQ0X-RBZjhUdIvy5?L0&1mD<`` z_-g?D^2*BO#QK!)TaRqvclvR2TpsiCwU-~Y$^kcfMxlWlN*m(>2q{a01D0d{6n2lw zpVNfL-UjX*ZwH=5_<%-q)Yvv+?!mpF!BmfJHIkUnRrj&6iv|S;=Q->d{{xWh-osK{ z=uSnOS4)L;bOxO~ns8BwyFfk5QLB`qedT4|3Na938}ex`zCSJJgm7$pA~Bg=tEQtv zW{D=~UKjo>AsC;AS0Xv;K-+_DdQ?qY+Y?49whJN{$eM{d+25G<^7HHaWRkHhl5^!p`DJx-;gm}_I0#dZ z8eJ2S&rp!3S;umsl@u3$MsCy!Ci)738zVW5uBfZ4qfYlqTzz2wZBlO{ZU(rQS8ZY; z!*}l7nas$kLPy!!+I|58r#y*pb*qUQbh!3)GlHwb{` z=qQMe8LvK(d{YVtB;iTeeQ^m1A+TgmV@WYiOfbJUHBHaOx%6H3_wo4-!n7Aqh~a^G zcpDi@_khdkfc@qJW1snXJ-HmU(Uf3UWJY1tprH@~Dmo^nuntsUjblyp%peS&#cvm>2(P;FJ%sRmOPX}9v$TfL>f;<)Vos>5(uXB`Z6P)A^-)#=EO`= z_B~H(a0UpKhpJGLAR;78Ay{2r_JM7WLIK`Pl1kCth8W`(J^f{CYioGU^79;&t!R(l ze1JYX&rz39#GG%#A37faB~8xnNXKWt{j84DXecEA{GC=b!@kTw5ch8JBjW{FDLNP% z6hsZP9mXL4l}_tTqYttD7MJP7yEHilG|DXH!XWrJZ?ER=R2l7~{SCw~aG?SqpnoF)8#L`jvLpnJJB!-v zYRbxy1$v#2K;ijDsohozaGrFUFf^a3ggxl`zdRUsl8AGHPRMD;R7$ zBZBF}z${-z(JN+0gLfRt4ie$q1e*P$tE($20Cw~I%3Q=kSF$$>7vkNQxf<_TKSTSM z3?DIg#8Q`2=g&#Btr9>XTn8<1G)f4}0!m7_^1EchnS~ntDVS)bqXPgTYC8aj7(US(;)+`B!ootO1WRm*)G2WI8T-8EfI2y@V-i#&pFi8%*<_|@Znohp%XNX z5#5U5wn$-3kz)wm-`}UBxQ`Nt5awQ4#N!Z>hb|{7KqwcSusaRc*9#k#$1^3pDth~z z-{%}I6BE`z$N(%7&Yqi$etv%Ff+Hg%1w}Z-tRjr6z<2cRK(`}_ z+7IVLCANIDC|Oy%CQg`C%aTTF&Qk&!4g*KRc)ANK1GciZ6&vo$?+`UEWVm0w(N{`C zOd@;}j9RNmtWH7)7`3Hwcmx)u3y51y%FSg?&NaS-F}12rl>OQc#-kCu|PcN|~T%1>pgedKEFtUJ8ZkSnE$WGm0RFFb8#>bBzBR>1w!gjz zZ35KFOydlE`er%*vk#T;_wOPR%$gsrGpVTz2}Jp{F05K1QExf_g*Q` z9>Wm4fgl^9q&K#=dm_Kk0ln;aw9#Zf-RLXT78o0QU*Pm8UoKPWc}qQvI+*Iayf&-t zA0qJwMdjo|j*hlEHi4@5j*E-yb25A!w?%g8n74*8tqo;X5#dN`93~rv_Fx zLqZ$S>2ql$^vqN#ePO@5Ob;|$Z{)zwFs3+myp{`7+YhZpcFfjCz_jY$b zTdxd79KERkv?~d*ZUM~=uuIwyY|}WUQnx;pqdq;I>=V%e!}b`Ww2F#~iHV7T?gpw> zd1&DUIlrr5rAkUmUxs)pXNdx+y>gxI%hm~2f@1Gzxy{p0a}ae^Xf@*EA%GW%8HCix zZ+i{+CmO+eU=Vd~cKrB(H8wUTI{nao`x1DE_4W1KeN|mpy0~~zl`fy?7)PsJC=3dR zl1fs1s8HfkQX(&2T&t+8jDL~dem@vS{+q0nl+Tz=^3wGD{3TeaiDnRyZEP>d0*hS- zt}TJ}4lV61Dk?Pi7@BIC5W3=DBM%@-YuhgOr;0qhKu%6h=zHsIrvc;l{rmTt!9ikx zvdoA1A%JiofS7Rx>r?*=5qtDK`!TkTdT7~Z(mNNf4U5NalG6)RT ze`g3oJWW5G6^8sPi%5qviSh3;0pwSnlh<8-^sieY-xD6X;k*6!Jw)s-(LzT3zphPO zX{0jeKi4eP|9ZCnrw=k8`n=NK3aEzHz#ed`UT*~nnQb9p5wJ?VB`Xrg5dQ$P%03G zi#>aW8_BN!0pNOlM+eiBHr#91J^)g_tCWiZzu(&B^m1CBelFwp$?+jHCJ=f3`nB{8 zck>!>7P{^|SY4|+L^9;XT-EJwnhF8kl|ReqwLLj1 zZLl*vf>C*%O?fu?H2X_A)k(k?pw~>OgW=Z(hT^3%He`O#D1p-b*>$Y6guFZrC8Z#$ z8F)U6GP;l-fMn&eRbHZ+f5!0cH2wjDf)yJlMdl3*D;`8dm39D{{y;+tb1Qqso+9FWi$UQQUiBa7LT2q}-{C9v)FAMt6K9d_S_ z+*iE`*sK9)sf{s*wOPx+kMkTzVl$fG=_VwKg&}vg-6#R@7dX1kN(UQg5qA;tDgG}H zk(g);Nm%M{dj_P5?;r6efB*IXn_qFbS`Lu9K$<(V(s}qR zInX@weOV-s;PL?KWw@eJorBZ`CMtw+W*y@WvJ+>`d99amd3kvOv>En=gohJVZo~?> z+y}d;sDRZT&CT|tb?a+pyMuh|pRbcw>MAn8T%!x7NJYG6mWCV+h~15pX!c#HLojAW zdbV2SSNyReqgQB1GGlXlp<5+vOwOQC`3m7(I@AnA)r51xpD$x$6Qa6OW#Xi!4HR&# z!9AJm1ML#Re*zSkfxGt3&TzCnbRUP57)l)GVip!LO)^V=M(&4zY3P0FWqvjLuT^su z!9ap);G+Ri=9*250&JT1T>?-KQ3Do4xM(QHKeJ(8zP`Pm5M&9foEpX-b~8uGI1)O+ zU!oqEC=~fp2#`b9M{#j+6bkbuyLbyD{0jhy-qu7Y27tPZ@4z^i0j=Oe@&Fs$rYK|# zV99yFR=kcw!QWGUx_nnx2l%=TD3?lv6cDhT2nUO#KrB~=2#|~>Y|Mn<#>g^V%i{wp z$K65QG{qcp$UV6S1mMETYe>_e?W=NF-K$dW`J`^l3w%j&-ECs=d(0)~@se7Q1Ih$Lul)6$ZR`rmQ72YeEwTpEzFG01vN z`92sEL>TQP34EzYb_u&1+B!Q>jZ+Vx2?@uc(kcy zNXb44-KV1*z==}<`o6F?94~xPLQ?XZ{t$po;;c2W9$Csl;lV2jgL|-Z8vsSqj+v%@q_T` zD(cU6x6BX8RPIidSZ=H_&nnnoR zsHwpXWTOHA8An7nUjMD;psvR8XEUy{va+_f)k6U{B{j8;HqMb6^&PT1KollFsI1!1 z{qZLPhPwI1KhEBj|9D$=wy_5=>{H2LK$ybXg3zcMfaE;>s%<~bZ?YK;v7=D}eZ2`4 z4sG$mo-;swZbE553oMx=m(!CMNJ`SOuSZp61|mec#qn?2PmobkskD~|cX$gKLMXt4 zP#=1GKJ5k{tVqB2D_(dOrpcWkb zT|$mep>u$p_{;|6hCW9oF;z|BpUx+40Wav{X_WO2a5L zjYxx{rL?r9QnC_CXlY54G-ykS&|Zp4N>f{m>$jvqfhWj&(T$MjM}LQ+EF8@fU9Mx^g!U(x{}*mE`u@r zQ(~CAc=Ib`j$fWS;~PpI@kWHkW1TkbA2wVa74YWcoQ-Sd0wN2UL#kP+>c zWokI4ZvTqT8&8;7|F^WCnf?Af!;FoOBT(F&XfN9o;G2H}+5(h;$t6FLW$2_%#u-J) z8QXZp&?8X{0};T?W?(eS=Ou>s;9$4i=w~D(CFyjuSHI7gWf$D-b?U$3J`zJibAvivd2X#^JuNNQabVa!PU80wEF_(mbwBBKj9ZBnCId+NxmBAoWm*(l zd%!5p%Mlg&6Z5KWazS7DcR}CPh@HtKnr}32di?nDCP~S!tu2E4ak7};{Par`Qa!b4 zo+o7m)p2hOgRGmCVK?BlDl;-6yaz%O97gj@yS`mU79z1J3{bQrxi82qQ!ZF+sylSa z7`hM3B^4`a@dEq)@V)=bE%tG4(NCjDSZF~2_c}Oe1|BpJYTL4NXS24aa3g#o^p43mwsrRm|;0j|GOQ&kR7G46} zZp|9gfiWeE!S=(>{omvO`L3C@a4py@cDJ6I z55tK+?TLrhWBD^i+1Yg-yj6KFepWfN2dCn zMpVGzXM82W(jE(?#};8r$|9^Df(Ss!xN;sOjh6!g0B&h$AiDH=X`!%X=$l{sAl&4KVGXkWBuBhqTuh__Ge!F z%`iZ0CKv3(_XQn;ktE=RY`CGl-4z{HK#QXs_el}c0KhM`wwfJ4^SyibG-~?WN<9#DPeD&fveo3bmR&}@|x6@)aVdcw#j1V&kJnKF1rJFeLdLhzJE|M=)usXxPf1gGOL>m}X zN6&BI$xFbvn>TOP)7YnU(fH$8)u(zQQ}k}Lgnl>kb+fd5TRZv0x*Hp(K&1l!T1`^S zL=l0BVIaZCff)gklu*%!2KQTA_KUV!+pD16Ch8Gtw~wR4rGa`%+B+dZfW&A4>+VnXyETTf(wWvEuv z|9}IW6+d3PaibZs2>B5=!MZ%5pY4KsJ?Hs+h*2!$Vl=9&Bl)JLagPWA#?Q z^HH>{xzcLyS-o?SaN~+~xBbg#i9%m+3Xh32DWA>?@AFt|y6tsQPOxy510g&LKh|8M z9$)Ve5I8UG;e&pH>>u~1PI-eO7S|;(h?dysy8gv6pAb2J5DLwRPBRCpYKMhw*z(Xk z&37@Xvuo+;xdUu&HQ!@y$OjHT(~_k2Y==$fKM*zD_4mI88Tfwliwd1_b_>h#0m*f? z`tAJ9VVh#*G_j=>3&+qhfBDwu^zTFRyT?tDaFw&5dZgd#!Z}Duhu2<810eItVy(ml zXH-#BJ_PN`?-=xxQ&4z^Vukru!m`JB1i8D4VIWWud7)tgC9}Hr%@glY?P!*{od#MF zYZVjwz_PpIggG=>y4!I1zs@zjox!}B6dgSi(9LN;5v1~GqQ?An#j4WUzB4^T52?ys z{GTe9)DfhZd4tAVKre4mnCMq@8Q0MV&Shv(du=%;&r=KaGtfT98GTH9u`a-peTCtE zn{Rq}h(;{{p_B!>$0kJ@idxibGnpjwdabTVOJ%Q*Cm_Wtm~Xka_Y*P*F3v?h;w`FO zFI}T8#^sQ0v%a~_#tUM&iUpKy?zeo;;)0KNznd-7jF){a^#0J~%5ugcpUR+)ystm{ zV{lLgdd?{fVb`M%3K89*F?|Ps5fymJv5xIl$gjv5*vWJD)xYyoiRm!*w@vI=RmD)( zjiX;bcM4|8qfD_#zjtCLTL$|o=5X9nME(Ilc&uUK^k8u9}N58k+adoJo6Q|n+j zjNao!PKH6?NfRvdzbD*W>h*U|%3h=R&$R??&N61|u9ZTD>w!CaA&7zo1G9oI!8jZ3 z=q(9{pNi&M!0}j_`_h-3em8qadtL0Qf6WhA;7~t9pi`w>eT@3{DeIvS@CGS$RX-&) z^>{zaoH_STrk|Gs6Xfa&y{|HG@J*AA3(`oiZi`SPB6t7miSdKPb5%S<(@i}E&nSdf z=}KKvc%@U;Lo7MjXkZ{F{% zkaEy&f~JhLKsB?43CEUN+Fj^_#-KCgFq)Vt>#;T~S7k!!z%qL_=~Km^LC*hu_JMu3 zBh(UJ1LNpv!Uz4J2!kT_VrWwC0|8y@@|B(Zc*1O8`4~Mgpc<$B+@%#tODV)uLMVvH z%&IRRt4a-JXl>w3YU-NW+Bi`sYj`dO-r$Kubk*+b3yZw~*}aF+Yac;yZHd%CZU3IK z!TfPlvphy9_uqrV2Xz*XThi!5BZ=1&^&W)y>kD>v+!Lx%M-xy|ASMZbF4r*a>geD` z39C^}B-+uP&nd-U^wpTy-C*i%^g(P81=KbmK-m+$D@4gvD1a2qjb83UcEB_LEnFor8n%btDUrn75bm zl|S``3}%P8_?C$UbLVK8WLi=4GX(cg@+cgrdfB~w>0y(d9!R)8v>_l0bbQw&%)5toyort(=;4&pkDxT zmh>eG>7roaF+ssfZKshA&X$XT7J}On`_6RyZQfV@8`eA#4wOOaYz7dcAsS1l9(q&y zTYs-o1gi*aG7VAALSO#Lmka$eAjp}VbuwBpOSw3FguDZ2V7*Mi z4s}+>@?c+-^jWU7f;fBy4qra{^r#&DtguBTj1gBo^-`XsyXJvh567<65anL8W(`Th z%ft3B{0+{2ewCT^GHLFeqqWUp(7uCI#)9HY&%&0mcVuzR*cON1m4H0}kl^$p0mDo3x7=PmrMFY){uv6TUr494}L z7cbb{h)tuo@>1o)U?Np`Llz74FS4h+ym1;huD;A`Ys+oT&dQkog2Rsf-BF=c_cQY! zWo+cH`no)&Fxng%i}spSekl0eungL)EwolZsrC*Mgg%&6!mvxlPOdJvN)la=(;-N- zY8e;evo2hp!?p)636bS&`Skv1=f5yHPkP{1h9W8Q4!8ZU9(TwtPb0d6Bd}Ip*54=iBvlTpL|=uWLC*USS6yM(L@zyXPG7Qkz+GO+v?f8T!NzkM*&&ZMyi z#}%8q|Nd!El5Y^rs3p3Hn$_BZYhglaYHqx4ZbMVk6Yz2p?@*D2w?WD)&g7$wd|7>{Ej{U-9^iK_~-M&j;a5Lzxw|Ixk>dB{rP{KW&L{@ z|Nn*jzmNd`pLsR^VY$5|L25w=vH&usCr6+1)_Az?idCJ)tX{2E>=961p=x(h^_TbY zmK^~9_sX1twC2c|TX*EWny%Sn^?euEO6HzTUb!(;MYZ_6`|*&k19OTWY_nNC8IoFT zPE3N6dLamFjpW_{bq>kjJ;Y;z+aSY#&;NStO0LJcHfvUkpWoFLgT&cDk6uem+`f9v zns|_AykFaZBRFXNwG51iHt+Z zF{uhkPY;9SW*8ywu8)twF+Zfac&BlD>)yIGz^tx#dM+Vw2JNCQa1gXFRdQBf0mncK z8J35$psK_zdmyNV;vPWPKRVW8=a+0#dMi_>$08}|l}R;cfUxmQ$UQuppP|Gv5-kqC zdUYCr*fOxB-@ham#*ClwMNt6X66-R4t)h-ly{C{8owK%ne{a+NL9?o{>YAD`KdRC{NS9ug(ixmnI$1drV0oMJOQYX(lY?g`oT`ytx)Kip5HDA9uH!KT$nYJ zQxKy`sE#k1jOA@@XZoagO{0-8iQt4!>f;wDR8K^jv&(z!0(ir_wF>MX*ppeYrcqH* zaO=-TyLkoGB8kN{8iCut1Mi!Z0wvXKe84*(EhMOKc=f9;@rB|_l-@iex7c&b?*A;f zBV{zlTYkpQ^kuJqAC0bnB=s$NA!kty)w*-lp$=?Ci!X08kqsrzCjt&v5fX zFJASNEMQHOYDJ!cTlwkw4@{N z6TAj2T`kdI{^G@p?>>0oiU$x1&=cbDrNlf|&|}1x>7bO(3zn892^3u+3J2&WcLhs1 zB9|H-OT*SEOFVNBnk#9vcr&q5iI4_IzXZLJ+ADiuH2xvpkx*YO7cp55HOO_;)En^% zQSAT)o-Z>UvWdj}rwYcM505!!-)}aQbI{)(EF_C|^d5{GD9AwIHe&@C)@6uca#xOf zVLu)!FhPD4cqkT0tpr3&7j70VjU+B|v*HF`E#_gLRQhoBHqF^XCFyqZ%%FO{itR%t za=cgL>u@tKCNmY7AIm4#CSZ4;MKsNyap+9(^BwySL zd^fJ(#Y6Fd^zQIB55cZ+d!Q(`*SgTOrCdTwL`di#tm?hEQVSq~boG$I74?swv2oD5 zc;Z~nL~3H9it6*oh=|=0M>_bfhS(nL-0KDOD$(At~9FX^ck zZL{q;!Mwqcgz)WR4}7z8a&!RFU4YQp^yS8R3?dM-t9SILkSMpXMZ!lYtX+UPfGa;} ze1$jN(5z_bmZN#$!$2yYz;FT%FfYkP*rv1~Q%51^3s34#xrT`XJ#+JgAkt!=W4-7A zrJoU;Uqs%6&2m;6cc%5tjdCrd>ttUp`e0<6&sP$pg5y2|O zdc)xTolS7Q>??E}8@2#&wH7$n^VHP+@IMgzMGo9kY1mfVKpFkReMH7@{mIu(AYPVi zfQn`h7ySCdyk7od)cPi0l2}N7@D@+$10<&Z@Tks1_s?}Cd^Sj4 z7bs?S%kb7cX3xc8O6NufjVLi7euKTMvY*hw@|u!iXpDq`*6+CaZ(so|_#edonx`RD z(hw5Rl5i?aLAfs!?dFrPc>>teDez-lFfJkQGL`e6A>$YBd zY2KK2Iy(nP6M(3FXxtic7qC#Bg8o~B-tHMWO*MP7yr&|Q<9+6`P$Kk`iB+ZUak?{b zs1k$Uk&%)9L%t6mnw2?8VYJDV^~hJz#RYK!r`|A-X_A2Xmg@l^bu*k-?>P6SGb1c4 zj1;!DU%%>uz*)@3#tyLE2d%gq9$q%Mcre?#Lr#K%5wX}2+>9Rd@FHJRHy|e84-DkB z?Ww&dX4UEnmB>GQ8b={h(K#}LMFiEV8~gz@7vPwFd}NP$^>Hl&rC`xYt&|>5*n?Q1 z_w;NgSMJnZFVA;B4P1sD@eN+$eoWe6VMq$9EUu*4HA;#D%;1O}M(0cz4j7rT0E5~W z7UHAB2xi$OCCZ#}(E36`A;aW>k4`E#`TTr0nz+H9cet^wS{YALPmFZxnw#?z?~Jw8 zS!|Ak!I5l%kcP(q{4uQ4Zd)y3qSO;3>-w>7l+1Bn2tb#Q1run-MWWqSZrHG)304KmbiaTJ;?l8* zXMTjqT>YP4098qUEBa7A;(oBxh@5%4u<4hh=Kjb7+!jh=PCiN3*&Vm?)J!6oyzkhq z5wgW`>*mefd-hC4?@iXxfNyrNlf`Ne2K1*QQv`gkQN6QP@)o>jGk^%VKvi2jz_0dv z{epE;Zdh^6sF5yX)(nWM4=|)rVuLV57Vq3Nv%ZLcR_0Lh*<)km0M=jdf7{ z#il7(sg)LSFis;`UW^}d5(Y%c8)(WtJUP7SRdma(W&C$AsfTIN8y+NM*eNk?ZR+2l}WfaW6#XAgL2dP=BdHi?U`1jtdP8Ar1X(AZ*=??;DE$Q>4(HGpu3 zpWm|*^a}xq-2VJo?czwsW5yZir$%jEE2_d7KXmk$GGl;mph#YU;R()wsy86<9zJ}S z#wNfJXg;(_xL8ctxxKx=@&^T-q z5m|~Hx)+h*2Y}P{TeiqQ{z^9vY^WKa6O}-LVph~t&4ac)uK1jajd#px{a7kf*A#Pm|xisf>R6=g+7) z!F46_S4fkBZ!t;ma8opjM!45c&M9n}x*nLeB{E!qR~w3MHnP=wtCR2^Cc7f!*%JOVD?Uo&Wwo?l;~S8C>uj zXEVQ1UMIgDA0MCLY*PE;FsJFHS_Cpnjq0D!*Ue>Eut{$MqOc>?|1O6ajo9S#tFY@<%ZBTUm+O-3aK9*f9 zx|xnCHdEGXU+lZ?nxU2!2sIP5jws7fL+xkskiNWx{LgE`E~D&(4LvD(To>D&b0=6T zMM_l9CV+s5d;0&@o-%l*j59XCii@xBq!sw^MGxI}Qu%`nW#s8t^A?hUbFD=y#-R36 zeda8nhiBaWIeybaDMy-@GWTDv88*!f#AVFdIia3>HwbEWhJga;4cV$8q@n^3?lV#z zl1><^&Chz;2z|0xv?wTH5Wcz2X?O|*lCt=G8fg?}^b5XU2%$D%0sweDQM>FA0iREj zJ4bHuJt(KZfUi6z#t#``GuW}+*nTu21ZaQ9`%4&n0MCXqM2=a|&;R`S6MfCmy5uM7 z-9Gow#AzfMh9pX2>d2ehw(@5Kp-g6e06lGg&wo5_HRP3%i6=cVVjvMPB z*1or)un0R^-*E#EwjZLzVRJQA-?G3lpqg_S3V%pFZmq`3wE(`RF`2(xSK;T73bTNn z;wDhXbx7(oOQc`F-FY8?^v|lxNRYJ59HDGxX3&c(`RLC8dMdwH*5dboz zfJfL-BKe|xe8n-t_^g7k7nbBJ7H}H7FxXqa4my-bRY@VyOhBC|x?{Ese2P-puyGjN z6LpOG4yelG&Ay6*4wy4BT0gN36Jxrz=y6~loxmgj)XMJ>LcF>ms>+6?anYhh&Ts~7 zM4$xNCQ|(vu5!foDw70p zrC{FuIGHSQuZ|(F>?wI4BUyQkjhzvu_0-!E7zN6>ej+85k-J=}VL?D!&;$#4S*HV}cG+OzXp{zI(d$s(xwx+9&{ga}Sy}KbE7U^UqncKcF)hX9%PX{8Pm&gvT${W!UWtUi||i zSmTJ>cqHHjZTfG!xqT|A`ls6BzcSB&#oumv9;&lx49Vb8rzDF@d&@^ih!lQ$UE6;l z$IWBR{`bLw=u4Vj>I1Hs=qb9coy4RhIBL#WT3$ia&K`YEr1X3r9uAI~qULqi6hSVgPyy(&JNsRvFA*BE2|4})^^_v zkf|Y~dgN_u_6w>xl#jC*0yJJl-++SlFlxAY%eOI5$C1exj8a0Zo6q136j@h!e!}tA z(9v;|tZ47kY9@L*t^ZE{Sm&gY`kY1}cS_ z4HG%uF7eI2Yua4CpIw6YxKZ$6&x>@bAqQ^}@`PgI2Iz1h^Saf~;sJn{9oFrO$m~G5CN{P13h2!27W=j!+igbWo5yG~-MO<5MKEq^8zd4Dm#FmS zDMpP_2DuExY|y@~;R1bE(pUDYXSb%X=59h=by+$~v$vCuSq2HrnOL&MmP<}ke3WneOkl(PN zXb(nUUdF@Qz{@)i-S|7up^Mh7^S~AApzJ|&o{}Jl38rEDbwRYfAX#GFtJ|39(F7IC z3}&#a>L5?t-!&Iwwxn3B&Zl>Xrp*>Cne1VT(5-Jo!pLN{jfjE`3f%hiXskws;fh)F zSB87$z>~`4K?uQHXa1kobeH)Hu~AgS_fUYZj3HM|3tp9K)w}yC;+EfB!LxO^_o5#} zQJ?Vi1n@aQjYvGYTOdB=0cu7lj{}l^p@-If8YPgawxhGChWHbZvMtn(iYKR*v) z?jm=0eKq_33;>cP(1uv+-i(1eS;K%AEkR|o_h8T(R1+Yr{d`p3k2UMbX+IFRd8%BC zwv4+GQmVx;-vsRd-cVri+Op+VeoIQY+rMq+kpT7k6KGb#Pi0-1>Cp_m7vC)h6ur<{ zZ2R&UFE6^n4C?vvP}#Y!EN;wS6{(`Fo6DbGGGD!#163R@4oLC}iB^gl;kS7v?RZ7s zkhjocH#KBkZO7OxdEc$`p#cPGxdC<98gc74fRG}S?KeM$^jB{ruiV#2l?@PzMn=6a z?489@h_p1+jwT5|QH$RQbbKL1xD*Bu?B7w20>B#Gby`l7mNkKGsHzUZUqGr+sLaVx zVSxvQ-^h7t8G9maF~NEO&Ai8^O`BGRS%_mm6x1+|qGXG(N@s|V7Yx z;gRU2y`~=#y#b(2BOW0=edk3-M;h$}1YtQi)N+RysAX=VVxma~^4eQq|C1VSm^PUe zh91RR`W-RY>sajdg8$ci7~zLUMnWNuCB~Lora&nDkY>x}MWK-%gZCVx_TtO`pe@RF zo-&OTub*PnRc;!~1&g-rP%mF5^=DWYF={3dOHdk#%7FmPbO8cCIUF_sZ)lBo(am{bqX_orv$!3!E3H@gy%wM<)lynO|zrG|50FLjLX50xe`VoTa zpfmC=T>?NP2k+bhixTCRp|@ETummU4ga-~sHC zEmLVukU{A~cSQIQ_>qWV))(qM?}&Lg3w$S|ik8io9UBS~R&KNyvzQ0lPEY-HZ-&~| zfp3+TtojjPRmcYvI63irV<@lwmjO{KuelBmBaxH=fk5!PJDN9@*VoR{IL$Rd9xONs zfg!`%4t$@MzRWW*DIG8REmy8teXLaMgj?D7e2{b2%{9Pxwtez zOr-uY*lwVeJ{H3K();6@_dP-c^>2JX`|J4i)2UhJr_1rb3b^$-#_P*McAF_j^s4}A?}E*x#%o^5r3Zw6kSHd%#Q z#)VY#&qCle!_5(LgdqJ6@ogw)6t$%{&F@P%8w`|p7sgAX!##UU>e*AYulVU71|IHE z{msKeM+$RNUdFCM8gksO({SUg=mE@l15-6UG&EFEd0-m=87_OS=22|$wp~rES z5qug@*3vJ5vYUJ=0*^<-1=7+?)Qtcf(?6i<*6J8@~g3+PJ%4?{V!6nfc zVTJ^?doNtr0a(iCT}06<6EQ7qUnwkpy*YoyCi+46JZhiRQcPB$wT)!MDie{oIJ`(V z$1MwqC$`S_^=oNRtFa|FT#G!xB4hPII?^8l4(1UgewZBQaqRbx9hr5OBav~jKZ8FG zt*?Q%^OJZz$j=$D+U&>svH<)BpKcb++p`oN?9`LJqz!WcL@opd4rFvxoQJcH*+p3m z-o@DGf5+Fg%EW7}$dJNPQFozmp`{B4etT`&b{G!g5;b%Yr_Whf>_#CvKN5moAyjeK zQRs)_=^0i$Wuqn>WM(4&q*{;0;T=c&=3BzvP?&ZqfZV^ zp(#|VpYbY)gFzTwgd$TGAj@<7a!~GTy^H)OQT|Jfe&3G43=G?rqleZyT453m{IRo9 z=SqhXCJ_!B4aMWwyGH2k%w(gBO%*A zMJ*xgE$CkzK*9yZtK2bSlq;HVDX-e2A+5*QlV`KBl(*0Yq}JKbg^+4Lo_jpp^;cgj zJ7JO?^6U+Cz9G0G((ME6c)Xrf{j5XpE#u31>b-Dx(0f$X-v!Om8~fb2Hg+5 zhYDZxLdEe2FR&nuw>e%6^9)FQd-2Yvm`??HmCZ{T91^jP-Z3PJ`S%3_tYWZV?%=s% zs?^+#p=>|VbR%ZZf?nVq^0j+q{&IzyPV{(as%8Q{yo~WmkDyj3ty<-#bs+GPa!yb& zX)I5Zy|!rE+lwRcxudNJ#oIRg^n?}b#szi0+EEULPDhN5#ADV$ro}YaACA9IhWJZ6 z_O2KDRT~|Z8dW8%;1MEb8lV}M*gD9qQ1he3DwMrSNAu)dt0oPcsgQPe5o?+883;XL zi9Z;;6<7`{BZyN0*vHV2LwLB6)Ul|6XEV^7mlx3R&Z)b4wLyijqeJ2OgS(jSjB0eR zu>+Wa8O|56?C@)$p2y)eQ;hTFc!u^t1DvBxKW_FOoF$ql46WbNOz30dLZc5cY`)9x2G0r1@Fn%Fu#89rkzHiIzsOFgShzL=I&}n;05r zbMw}%`(QZRuL+ygs_{VMVn#!w>MWPScZ@bG@_of>TgOY_ELEHMv@$+7^!-5tVADwx zAGy^N!s~P~s6qFLruuzf*F0eR&r$sgLC&;qgNq4UO&o-b`V~vjDC%YNSyCT^$cY z^&6@hy*)Nnz`vRJ2E&%G;H_`tlxG3(@;-Pst76))LKUChmhu-E?za;a6H`Z`-N=OF z9>qI~Ol@_$4VA~0A4+heHRNy~_x&XaA5>5VRM7gUKcHXN0c`nkjtaEw0Ec5UOvKED zho-EDSid+yFu);(@@1+bTDuhHF7|)cx0I^f`zbFSe#O<58Z+TQ(k*}Y>ti0oPPfe0 zpwSI9QlPK5gMNQ75|RZP2IY5>HVk>=hc5i?1>vGG8vm`1OVOq63m zxyUD5<)m*KCbx}Gt7;j$y0%x)sF={ILpdyOiBtxzL&H8;CJ+=s?|ezhZ#F4iX!>HujvX|glGF}RyYhkd4o0oDuHX;7WcJL~nJGa* zL0(*k_c)CY9sdoSWV3|C2_X2JxwsB6d6-M;l>tjUJLE%mL73=&iCZzS5MKZfV?JFL zA}86Dr>ut%SHlQnabv<)2aqF~M3;t=yR*DAVqp`ZaoMjJC13*Uiu<1{N z6WoG!Cy|`~%mH+J5oojf!C)6aL8gZQK;DD__F=rT#sB=X8S%&m>AL_CN*}FvIWi6V zI(HP;`V#Y3?>O}K-8*mT;?2GI-S>Fl%mE-t`p_U-!qBPy%079p=d0$-g==F}7J_ym zl8{$DL53~?QRqGmC-|XrMFR7ueTwppk$|S|0+;d)qAo9#2Zg8vBftz5xXW2_&YnM{ zY0NRZILcG3nOb%GQr!`7*b2CSY}gXyy|_YF+ZKyu1v+ikNpCik4;IoxV4u1Rw6-4F zj|2oQdfC=Jwbc!MqGwD3j))kSo}tV$o&;SqTtk?z8F(B?j1v*%KD5X;p>V)&Zu)4c zf4;dK&Uz|EK-%7>1%sM^6KjZQOQ^RsjA+&s^9_?L?$R>`1J8@3;RQI4CpX`VqX|jO+I$X)2IBrxfg-IqT z#}+e9jg1><@gde0V04i_autrZ5X0?@rYrh<+KIsdwD|rI6&Bm2d*Q+hGA1xzS@tND zylT~|;uUjgO)@omS>pfYF|X-f$2J*-xa^b{OAT6*U0hwY_()N2KVLs^{Xf57uUZTe ziH-Fh3M5ycc(CoS2wblyFJB&D3*QrFd*lHt4M~{;fBJoB}}(Z#U8 z{Fj1d@v>(qM-RXqrm6-sdWLgl0jpKl<7!)2R@{8LQFD7jbLe{G#tqOAh1g0qO=r!9 z2`1v?6%aTKwx#o@zqaUq)A@&C0`Ssb^vWRyPY)dhv2qeqY zU0qd~{;Sw9wI){C$zMAes=2bJVISp1m=vf6$T@~BQvG);9w7OJPow8w)R5iDvi*C6lylnga5QoW@Lb#mKMTXNj>$J&ZHv)+Y9M)}%fwNkIcO^}^xfAmrHaPr}R_48rj?^{F(+ zFu{F=KOqYa1ug{CJQuOd+fpABHP+%Qo1x6ziwr}?Kk73uPKb21M3j)#MM_GG=OlCk zjlI3kJhd@Ud#iS)8;bR21VD|`fW-F|j#~af_51NlYjg8elsr?ND@WNqR*Ox=bas4G z*SGEHMVUAFfBEj1MmRa?S!(9$h#D{0v(m@wp7OXj2>9oB^ zTl;mgibveAe6tef_=E9rK{n|A*$DnDOU zbr7S+7A9-BqtXrqPFC>yC16+r#W|D%XEQHgzRMrl{{EA3cqHHGV8=A_6%PaGCyRga zmuIc^80U2)7G6ry$*M&5Ru|tATtTlC3-%P=!xQb-QV6Nn&<$bu?qVb-gG(X;n-h&+ zu`5dEpq{w_2?b5S$IrKCIH=SMzdM2161xGSV7U{i@1Q;%%%vlTh z1QLm=Z@y3cB`t&ev?~8D|0o-Q$EH@zHX_?Ay{bPXXV)6X^Tl1Swus#T%=zkG^B){Ob?Q-xNU_o|h*8h}53^^_ z{!rc8+39bjhM1pFTt4E>b@;7eH38E23CeTkC-C~vRk+S3iKNgoki@b4V=h05R^9}C z(d4mTW&77{z$AZ~`h{#p=t&TZ%3gUNUL$JFRi-03x1pB+TwlDFKe9DKH5!6S4{D{>RQU}# z?&VcF<0jUv8{Kz$tLD#5Z&Doh1iR(dXYKyQv*+Bo=XF9UYilak@+;Jw)Ruk)E{_4Y zKTF)xdxgHlCA9)tbXqPYRU>yi1u$*nEluL0k!W@NSa`ns^GFwKTJ)x;6pr^OlLYqfv{g z8wZ_ym2b|$GxD9G;LKaHuo(Cn_*tf9i8=`>;Ihv;b>XB_!& z^sFvYQ!&_Ewbbg_WI0fkhI0rlr3or+Va3|cdbIm0&5FhtQ&!M3g3CpLst_}Q2SLiZ zN5J86@`MVm&WuTgXNTZHhh*NaODCB6I-&-npDxX&jZ}a6FJ$(Ly+sC4k zX(Q#IElo5XGI&jG9mkJam8{xvZashzgLA@EJLKJH`2p=S_fIN3P zg4MJ~my$F23d4^?4c@u5YeI|;tAd=wU-0q?z*s1)-oVPT@d_HL8WY78JI(!63>2*mx-!QGkV3n`o~P2FeJ{0cjb$O)HbqL(UgPjkJWL)>;L- zG0_c)ChIC!N8@#Ez-=O;gJP5Zf$h0sJnrr>03*zDDRV=PV4*(V2*2g_!#_^PaeJom ziv2RkwJDy=vwv4gn~W6-$Bd<|*;WFqtQ}38_T6;tcf}Cjw|~zm@{3!W0rLnc>u|L? zfOm+C?|+bjQ;!1Z`512D3nHBLRXta7dF}2ej8w19LT4v+yt8FO9q~$eip;_*(1!YQW+$@1G8f zbwF#32@UVj$~;Zxxgvu0hZAS${e29D)-@d`I#A)z&^V5z}4v@D_4NvyJ? zFF`*Y8lx2$vr>qe!IceZ8ud@F#e^Ca0*2|jCVA=4YH-+sJ4@X>L=@!YvfzfD0x22_ z_l*jLM^PqWXvBuCTa}F+P)hgj`0R#4bp~q|jT^yh3;}XS@0Ca|l(OJN0*-1KKhQKV zX5-WNhvF8P31TcsN&<0V{BI{A(B5-*Ib1Z*Pd{TjZdricaSoknSoAE@ta9CCv{Ph= zLHU}Ctw$3@P-Zux1X0(}axC!WU&wq$&sdi~Oj9b-_;I_<=5A*161%Cb1M>H$6pJ<0-bbf-DJFW)Cv;bR~|8ry{~>T z85b^Ap@8(>0mB}8!)#1#@M5cYegN|naHI;XT2KyqcLV!Q`t3foxZoN@$Hv{pKnFBD zQW9PGa&M#+b?;B{QV$jihvbRTT75}r6bA;;aCKSpl~%ba@d?#U7pfD!VxZM^usr0g zYX6eVL(>qh-JFfdkIpn#1$Si7V(bpUDils8@{Ca?$=J>XO(!n%p?dGZk5C&<0X}dU zDOm^AC^^1q&}F~JQC>dJ*41PiZ^`ARS*_sCsSv^*F{Y`tC>*0rq|pTR$JNFdnECPb zp-n#xJ5zz~cjsl}O50h2kF~`h<~1Q;t`*kWeiO2 zWseW8gT?@x`p+&9E`2x~v=QjXazVWVK0e2Tarrb`6b&yj2it z3+XTgxv;c2q=}+$<2ZbC8k!K2Es3B1{$!{m1-uCzaaJIrP~f%ZzXve&Hy45no(cxo z^QBaXiSa+Yebe>8-%Pgwl_k*r;z8COWCn*AgNQ5|tqC*;e)(+>6yL^s4Z-`UKH3y> zZ_N$Tsg^*zpJWpeL$ZiQbacOR;96*0MZVbMao4{VeV4EX(DkzOXsk5A@ zVBxI$oMy!d83!xoETdE=5ys?%9m<^;$=qI@_}DUIpr!Bz=(f(2vA!x&kX?{`7Q2@V zI%7DfFzMhr4szh~kWH-6JrWHQ8L%U&r)IOT>_)b2#J}T?AEC$>slJic9EOIOm=$1T zF?FeF<9R-UU7jv9dK8NHnY5B|Ay8S6UKKRG4E8yWy|Oh~#S_3+awf&Z7Kt2O63IVF zI|mJpJ6T8p_MEe^kp&cpb6AA{B?ovv69(M14RR6)kQIZa)G@%GdtOhLQSR zeUvI?uVY)W8?XBME~As-AV&nKWaO@{nVK#(v=cL)kgyUjwwzQQ1F0h{MaUg6Q9Zj;X{ObAjRRY(##iE+`FW_-t5EWYIW?e!oxPn;_D=g3(8q z`u6D9vSP7FNMH(VgWSdstiuzA?j{RAgEcP;%Lc}rELy&NHjYoZI8<@Gz@e6xTjl)o zZ?yqys)Vh+h=^*PnbPdICAz?*Gx@V>+qds8D#7hUUnZeKcohQj9psZeGUtGQ6Xgv1 z6KSOJOuav5Cu*Gk^b8O>sc%rH&ZohJIOSlI^D-RO0tYHdfyj`9!4tG6X>>cDGo&rq z=(Ekrr7(`(ncmFrjSe7PYBAOst?~r~yXU|-TfW}%gE(yksX~!iYcuSYz_-Pv8nYa{ zR_wr5SvdC)NoN6A-h}~m7{3Sg&2kD&?-okGNB^5#mw4K2EN~FVPLXpEy(|{?!gKg0 z?A67z{#J^ZJXunYVNa-iWl`=@dkrxLR1tg9exJs%h!kg0qr+n7YMB9HItl#o%(taS z8y*6@M5nFM!FA|Raa!bBwDm4PV06IMG9rZh$QzsU$^2vDdauu0#Nx9y6WPXb z34JJ*aVXV%JQ~`V*!w0TA*N4JZ8QSemw{XHE^N@$w8Feerws$%H97nc)dNOn(c}!0 zPy9^J#i9UZ*9Y*N4lIRg|cLosRsG1{-S(BK6GtistKjuz!?s~9C4PZxXEsuECp-w^$npwjMMGc_&>u7cnG!!#g0dQXm!QOZvWVASQxM^-U zn0svgMXa!%Dn8EOr@|L`3MV}v!aV4Z3Fht-7S&SfUX}ADPfZJGBU#ar32o}wzyn&0 z{xpRaa!Us!N{gV90j!9(Zk)Z0wf2?6GE{1dSSjNIrGFd)wD$t+=g6&*D!+>E3?W*R zFBXNy84YQL{sOM8TNlv7f%g?9{0)NI!aSrvaV??4$~%;Fl}`%7a+n1fAnT$<43atC zJvIKwBeUnsNstQ?T(xfFRJ;r17lam-w&9haGwrC2G}*5*dRTQe{u?d@765j$J%|nW zq%S$OXwZM@a1qpXB^cpbTt49E64h;{EUg58oSxohR0x2;VZ70^v@@&E#K`37IBI*9 zM)Wz1i@%HpDFk$kxXndqF;Gi+duQ-3VQu+>v!-W**SgxqwRg0%9XE*sJ+9-_WHKYF zO7e1hx8$o33i9b9eKcpsf*=w15QKX>I;^@QnE zG}Cd`=yEz@J)&$1Vjw zxAr$=rytVF&EL&@#<7)ft8Rmxck=j*R~$H*La*F&kro?Z;&(XH{7wFDQkPau|2zEW zfWX;LT&R4SVIwXWRYh5c;fZ&v?5`U^68It0(IM8>9N$?sf#&#XR<0t=;4O>3;jx?@ zI`!rvyIIRx*3gCn&;Vc!YyNIWI@jcI0$g)@1rC|RxBb_jP>J_jAJ2ux@W1`3zVUaW j?|=Q(|Ehyb`X`v#`?F&@zvW9X_#-E)EEBiq)Rq4Yz+|_B