From de3676a2d0adc2df6396e4890413861df1e840b4 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Sun, 6 Sep 2026 10:13:35 -0700 Subject: [PATCH 1/6] [None][test] Pin the float32 precision of the attention-plugin rotary table ``RopeEmbeddingUtils.create_sinusoidal_positions_for_attention_plugin`` rounds ``inv_freq`` to float32 before multiplying it by the position, so the absolute angle error in the stored table grows linearly with position: negligible in short contexts, about a milliradian at 32k. Add a CPU-only characterization test that pins that behaviour with measured numbers -- exact at position 0, below 1e-4 under 8k, 1.5e-5 / 1.3e-4 / 5.6e-4 at 1024 / 8127 / 32768 within 4x bands, and linear in position -- plus the bound the alternative construction reaches (inv_freq built in float64 and rounded once, flat at ~3e-8) and the returned dtypes and shapes. Building the table in double precision was evaluated and deliberately not adopted: reference implementations round their own frequency table to single precision too, so making one side exact removes half of a two-sided rounding difference and changes which near-ties flip rather than improving agreement. The test documents the accepted drift so that any future change to the table construction is made deliberately and re-measured. No behaviour change. Signed-off-by: Brian Nguyen --- .../test_rope_frequency_precision.py | 155 ++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 tests/unittest/_torch/attention/test_rope_frequency_precision.py diff --git a/tests/unittest/_torch/attention/test_rope_frequency_precision.py b/tests/unittest/_torch/attention/test_rope_frequency_precision.py new file mode 100644 index 000000000000..472e41fedf85 --- /dev/null +++ b/tests/unittest/_torch/attention/test_rope_frequency_precision.py @@ -0,0 +1,155 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +"""Characterization of the attention-plugin rotary table's float32 precision. + +``RopeEmbeddingUtils.create_sinusoidal_positions_for_attention_plugin`` rounds +``inv_freq`` to float32 before multiplying it by the position. The stored angle +is ``position * inv_freq``, so a relative error of about 6e-8 in ``inv_freq`` +becomes an *absolute* angle error that grows linearly with position: invisible +in short contexts, about a milliradian at 32k. + +That is a characterization, not a complaint. Building ``inv_freq`` in double +precision and rounding once, at the end, was measured and deliberately not +adopted: it makes this side exact, but reference implementations round their +own frequency table to single precision too, so removing one side of a +two-sided rounding difference lands on a different set of near-ties rather +than on better agreement. Below 8k nothing moves either way. + +So these tests pin the *current* behaviour with numbers, and pin what the +double-precision construction would buy, so that a future long-context +investigation does not have to rediscover either. If the shortcut is ever +removed, ``test_the_float32_shortcut_drifts_with_position`` is the test that +should be updated -- deliberately, and with a re-measurement, because the table +is built for every model. +""" + +import numpy as np +import pytest + +from tensorrt_llm.functional import RopeEmbeddingUtils + +# A 64-wide head at theta 5e5: a representative long-context configuration. +HEAD_DIM = 64 +THETA = 5.0e5 + +# Measured deltas of the stored cos/sin table against a float64 reference, +# max over the row at that position. Linear in position, as the analysis says. +MEASURED_FLOAT32_DELTAS = { + 1024: 1.5e-5, + 8127: 1.3e-4, + 32768: 5.6e-4, +} +# What building inv_freq in float64 and rounding once achieves instead: flat, +# and at the limit of what float32 storage can hold. +DOUBLE_PRECISION_BOUND = 3.0e-8 + + +def _reference_row(position: int, dim: int, theta: float) -> np.ndarray: + """The float64 cos/sin row at ``position``, in the plugin's interleaving.""" + inv_freq = 1.0 / (theta ** (np.arange(0, dim, 2, dtype=np.float64) / dim)) + angle = np.float64(position) * inv_freq + return np.stack([np.cos(angle), np.sin(angle)], axis=-1).reshape(-1) + + +def _plugin_row(position: int, dim: int, theta: float) -> np.ndarray: + """The row the production table stores at ``position``.""" + _, table = RopeEmbeddingUtils.create_sinusoidal_positions_for_attention_plugin( + position + 1, dim, theta + ) + per_position = dim # dim/2 frequencies, fused into (cos, sin) pairs + return table.reshape(-1)[position * per_position : (position + 1) * per_position].astype( + np.float64 + ) + + +def _double_precision_row(position: int, dim: int, theta: float) -> np.ndarray: + """The same table built in double precision. + + ``inv_freq`` in float64, the angle formed in float64, and a single round to + float32 when the array is stored. + """ + inv_freq = 1.0 / (theta ** (np.arange(0, dim, 2, dtype=np.float64) / dim)) + angle = np.float64(position) * inv_freq + row = np.stack([np.cos(angle), np.sin(angle)], axis=-1).reshape(-1) + return row.astype(np.float32).astype(np.float64) + + +def test_the_table_is_exact_at_position_zero(): + """Position 0 has no angle to get wrong: every entry is cos 0 / sin 0.""" + row = _plugin_row(0, HEAD_DIM, THETA) + np.testing.assert_allclose(row, _reference_row(0, HEAD_DIM, THETA), atol=1e-12) + + +def test_the_table_is_accurate_enough_below_eight_thousand(): + """Short contexts are unaffected, which is why this is easy to miss.""" + for position in (128, 1024): + error = np.abs( + _plugin_row(position, HEAD_DIM, THETA) - _reference_row(position, HEAD_DIM, THETA) + ).max() + assert error < 1e-4, f"position {position}: {error:.3e}" + + +@pytest.mark.parametrize("position,measured", sorted(MEASURED_FLOAT32_DELTAS.items())) +def test_the_float32_shortcut_drifts_with_position(position, measured): + """The stored table is off by about the measured amount, and no more. + + Both bounds matter. The lower one is the point: at 32k the table is wrong + by 5.6e-4, roughly a milliradian, four orders of magnitude worse than + float32 storage. The upper one catches a regression that makes it worse + still. + """ + error = np.abs( + _plugin_row(position, HEAD_DIM, THETA) - _reference_row(position, HEAD_DIM, THETA) + ).max() + assert measured / 4.0 < error < measured * 4.0, ( + f"position {position}: measured {measured:.1e}, got {error:.3e}" + ) + + +def test_the_drift_is_linear_in_position(): + """The signature of a *frequency* rounding, not a per-entry one. + + A per-entry rounding error would be flat at the float32 epsilon. Growing + proportionally to the position is what says the error is in ``inv_freq`` + and is then multiplied by the position. + """ + + def error_at(position): + return np.abs( + _plugin_row(position, HEAD_DIM, THETA) - _reference_row(position, HEAD_DIM, THETA) + ).max() + + near, far = error_at(4096), error_at(32768) + assert far / near == pytest.approx(8.0, rel=0.5) + + +@pytest.mark.parametrize("position", [1024, 32768, 131072]) +def test_double_precision_construction_is_flat_and_at_the_storage_limit(position): + """What the double-precision construction buys: 3e-8 at every position. + + Kept as an executable record of the alternative. It costs nothing at run + time -- the table is built once when the engine is constructed -- so if a + long-context investigation ever wants it, this is the bound to expect and + the reason a wider intermediate is enough. + """ + error = np.abs( + _double_precision_row(position, HEAD_DIM, THETA) - _reference_row(position, HEAD_DIM, THETA) + ).max() + assert error < DOUBLE_PRECISION_BOUND, f"position {position}: {error:.3e}" + + +def test_the_returned_dtypes_and_shapes_are_unchanged(): + """Whatever the arithmetic inside, the stored arrays stay float32. + + A change to the intermediate precision must not also widen what is + returned, because both arrays are consumed as float32 by the attention + plugin. + """ + num_pos = 256 + inv_freq, table = RopeEmbeddingUtils.create_sinusoidal_positions_for_attention_plugin( + num_pos, HEAD_DIM, THETA + ) + assert inv_freq.dtype == np.float32 + assert table.dtype == np.float32 + assert inv_freq.shape == (HEAD_DIM // 2,) + assert table.shape == (1, num_pos * HEAD_DIM) From 837166275fe86613124d6558f30e0dbd601a7cff Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Wed, 9 Sep 2026 18:14:58 +0000 Subject: [PATCH 2/6] Address trivial review comments Signed-off-by: Brian Nguyen --- .../unittest/_torch/attention/test_rope_frequency_precision.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unittest/_torch/attention/test_rope_frequency_precision.py b/tests/unittest/_torch/attention/test_rope_frequency_precision.py index 472e41fedf85..f7be599fd8ac 100644 --- a/tests/unittest/_torch/attention/test_rope_frequency_precision.py +++ b/tests/unittest/_torch/attention/test_rope_frequency_precision.py @@ -77,7 +77,7 @@ def _double_precision_row(position: int, dim: int, theta: float) -> np.ndarray: def test_the_table_is_exact_at_position_zero(): """Position 0 has no angle to get wrong: every entry is cos 0 / sin 0.""" row = _plugin_row(0, HEAD_DIM, THETA) - np.testing.assert_allclose(row, _reference_row(0, HEAD_DIM, THETA), atol=1e-12) + np.testing.assert_array_equal(row, _reference_row(0, HEAD_DIM, THETA)) def test_the_table_is_accurate_enough_below_eight_thousand(): From 1de4c9a52b681a38678a5d31c90b1314adc48857 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Thu, 10 Sep 2026 06:41:50 -0500 Subject: [PATCH 3/6] [None][chore] Document the fp32 rope-table precision intent at the builder Per review: make the fp32 precision behavior discoverable in the code, not only in the characterization test. Add a note at the dtype knob of create_sinusoidal_positions_for_attention_plugin explaining that the fp32 default is deliberate (the stored angle position*inv_freq accumulates the fp32 rounding of inv_freq, an error that grows ~linearly with position to ~1 mrad at 32k; fp64 was measured and not adopted), and pointing to the test that pins it. Signed-off-by: Brian Nguyen --- tensorrt_llm/functional.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tensorrt_llm/functional.py b/tensorrt_llm/functional.py index f39b6eac7a03..6ecacd482d57 100644 --- a/tensorrt_llm/functional.py +++ b/tensorrt_llm/functional.py @@ -264,6 +264,15 @@ def create_sinusoidal_positions_for_attention_plugin( # Other scaling configs that only used by certain scaling types. rope_scaling_config: dict = None, duplicate_data: bool = False, + # fp32 (default) is deliberate. inv_freq and the positions are built at this + # dtype, so the stored angle (position * inv_freq) accumulates the fp32 + # rounding of inv_freq -- an error that is invisible in short contexts but + # grows ~linearly with position (about a milliradian at 32k). Reference + # implementations also round inv_freq to fp32, and building in fp64 was + # measured and deliberately not adopted (it removes one side of a two-sided + # rounding difference rather than improving agreement). Pinned by + # tests/unittest/_torch/attention/test_rope_frequency_precision.py; change + # this default only with a re-measurement. dtype=np.float32, ): if scale_type == RotaryScalingType.linear: From 2193d0043907287d8b2a48b61eab352ebe0bc465 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Sun, 13 Sep 2026 23:19:08 -0500 Subject: [PATCH 4/6] [None][chore] Pin the rope-table fp32 dtype as an internal variable Per review: no caller uses a non-default dtype, so remove the dtype argument from create_sinusoidal_positions_for_attention_plugin and pin it to np.float32 as an internal variable with a comment explaining why. Drop the now-unnecessary characterization test that pinned the fp32 precision, since the dtype can no longer be changed by a caller. Signed-off-by: Brian Nguyen --- tensorrt_llm/functional.py | 19 +-- .../test_rope_frequency_precision.py | 155 ------------------ 2 files changed, 9 insertions(+), 165 deletions(-) delete mode 100644 tests/unittest/_torch/attention/test_rope_frequency_precision.py diff --git a/tensorrt_llm/functional.py b/tensorrt_llm/functional.py index 6ecacd482d57..4258696f2c83 100644 --- a/tensorrt_llm/functional.py +++ b/tensorrt_llm/functional.py @@ -264,17 +264,16 @@ def create_sinusoidal_positions_for_attention_plugin( # Other scaling configs that only used by certain scaling types. rope_scaling_config: dict = None, duplicate_data: bool = False, - # fp32 (default) is deliberate. inv_freq and the positions are built at this - # dtype, so the stored angle (position * inv_freq) accumulates the fp32 - # rounding of inv_freq -- an error that is invisible in short contexts but - # grows ~linearly with position (about a milliradian at 32k). Reference - # implementations also round inv_freq to fp32, and building in fp64 was - # measured and deliberately not adopted (it removes one side of a two-sided - # rounding difference rather than improving agreement). Pinned by - # tests/unittest/_torch/attention/test_rope_frequency_precision.py; change - # this default only with a re-measurement. - dtype=np.float32, ): + # The rotary table dtype is pinned to fp32 and is intentionally not a caller + # argument. inv_freq and the positions are built at this dtype, so the stored + # angle (position * inv_freq) accumulates the fp32 rounding of inv_freq -- an + # error that is invisible in short contexts but grows ~linearly with position + # (about a milliradian at 32k). Reference implementations also round inv_freq + # to fp32, and building in fp64 was measured and deliberately not adopted (it + # removes one side of a two-sided rounding difference rather than improving + # agreement). Change this only with a re-measurement. + dtype = np.float32 if scale_type == RotaryScalingType.linear: scale = 1.0 / scale if scale_type == RotaryScalingType.llama3: diff --git a/tests/unittest/_torch/attention/test_rope_frequency_precision.py b/tests/unittest/_torch/attention/test_rope_frequency_precision.py deleted file mode 100644 index f7be599fd8ac..000000000000 --- a/tests/unittest/_torch/attention/test_rope_frequency_precision.py +++ /dev/null @@ -1,155 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -"""Characterization of the attention-plugin rotary table's float32 precision. - -``RopeEmbeddingUtils.create_sinusoidal_positions_for_attention_plugin`` rounds -``inv_freq`` to float32 before multiplying it by the position. The stored angle -is ``position * inv_freq``, so a relative error of about 6e-8 in ``inv_freq`` -becomes an *absolute* angle error that grows linearly with position: invisible -in short contexts, about a milliradian at 32k. - -That is a characterization, not a complaint. Building ``inv_freq`` in double -precision and rounding once, at the end, was measured and deliberately not -adopted: it makes this side exact, but reference implementations round their -own frequency table to single precision too, so removing one side of a -two-sided rounding difference lands on a different set of near-ties rather -than on better agreement. Below 8k nothing moves either way. - -So these tests pin the *current* behaviour with numbers, and pin what the -double-precision construction would buy, so that a future long-context -investigation does not have to rediscover either. If the shortcut is ever -removed, ``test_the_float32_shortcut_drifts_with_position`` is the test that -should be updated -- deliberately, and with a re-measurement, because the table -is built for every model. -""" - -import numpy as np -import pytest - -from tensorrt_llm.functional import RopeEmbeddingUtils - -# A 64-wide head at theta 5e5: a representative long-context configuration. -HEAD_DIM = 64 -THETA = 5.0e5 - -# Measured deltas of the stored cos/sin table against a float64 reference, -# max over the row at that position. Linear in position, as the analysis says. -MEASURED_FLOAT32_DELTAS = { - 1024: 1.5e-5, - 8127: 1.3e-4, - 32768: 5.6e-4, -} -# What building inv_freq in float64 and rounding once achieves instead: flat, -# and at the limit of what float32 storage can hold. -DOUBLE_PRECISION_BOUND = 3.0e-8 - - -def _reference_row(position: int, dim: int, theta: float) -> np.ndarray: - """The float64 cos/sin row at ``position``, in the plugin's interleaving.""" - inv_freq = 1.0 / (theta ** (np.arange(0, dim, 2, dtype=np.float64) / dim)) - angle = np.float64(position) * inv_freq - return np.stack([np.cos(angle), np.sin(angle)], axis=-1).reshape(-1) - - -def _plugin_row(position: int, dim: int, theta: float) -> np.ndarray: - """The row the production table stores at ``position``.""" - _, table = RopeEmbeddingUtils.create_sinusoidal_positions_for_attention_plugin( - position + 1, dim, theta - ) - per_position = dim # dim/2 frequencies, fused into (cos, sin) pairs - return table.reshape(-1)[position * per_position : (position + 1) * per_position].astype( - np.float64 - ) - - -def _double_precision_row(position: int, dim: int, theta: float) -> np.ndarray: - """The same table built in double precision. - - ``inv_freq`` in float64, the angle formed in float64, and a single round to - float32 when the array is stored. - """ - inv_freq = 1.0 / (theta ** (np.arange(0, dim, 2, dtype=np.float64) / dim)) - angle = np.float64(position) * inv_freq - row = np.stack([np.cos(angle), np.sin(angle)], axis=-1).reshape(-1) - return row.astype(np.float32).astype(np.float64) - - -def test_the_table_is_exact_at_position_zero(): - """Position 0 has no angle to get wrong: every entry is cos 0 / sin 0.""" - row = _plugin_row(0, HEAD_DIM, THETA) - np.testing.assert_array_equal(row, _reference_row(0, HEAD_DIM, THETA)) - - -def test_the_table_is_accurate_enough_below_eight_thousand(): - """Short contexts are unaffected, which is why this is easy to miss.""" - for position in (128, 1024): - error = np.abs( - _plugin_row(position, HEAD_DIM, THETA) - _reference_row(position, HEAD_DIM, THETA) - ).max() - assert error < 1e-4, f"position {position}: {error:.3e}" - - -@pytest.mark.parametrize("position,measured", sorted(MEASURED_FLOAT32_DELTAS.items())) -def test_the_float32_shortcut_drifts_with_position(position, measured): - """The stored table is off by about the measured amount, and no more. - - Both bounds matter. The lower one is the point: at 32k the table is wrong - by 5.6e-4, roughly a milliradian, four orders of magnitude worse than - float32 storage. The upper one catches a regression that makes it worse - still. - """ - error = np.abs( - _plugin_row(position, HEAD_DIM, THETA) - _reference_row(position, HEAD_DIM, THETA) - ).max() - assert measured / 4.0 < error < measured * 4.0, ( - f"position {position}: measured {measured:.1e}, got {error:.3e}" - ) - - -def test_the_drift_is_linear_in_position(): - """The signature of a *frequency* rounding, not a per-entry one. - - A per-entry rounding error would be flat at the float32 epsilon. Growing - proportionally to the position is what says the error is in ``inv_freq`` - and is then multiplied by the position. - """ - - def error_at(position): - return np.abs( - _plugin_row(position, HEAD_DIM, THETA) - _reference_row(position, HEAD_DIM, THETA) - ).max() - - near, far = error_at(4096), error_at(32768) - assert far / near == pytest.approx(8.0, rel=0.5) - - -@pytest.mark.parametrize("position", [1024, 32768, 131072]) -def test_double_precision_construction_is_flat_and_at_the_storage_limit(position): - """What the double-precision construction buys: 3e-8 at every position. - - Kept as an executable record of the alternative. It costs nothing at run - time -- the table is built once when the engine is constructed -- so if a - long-context investigation ever wants it, this is the bound to expect and - the reason a wider intermediate is enough. - """ - error = np.abs( - _double_precision_row(position, HEAD_DIM, THETA) - _reference_row(position, HEAD_DIM, THETA) - ).max() - assert error < DOUBLE_PRECISION_BOUND, f"position {position}: {error:.3e}" - - -def test_the_returned_dtypes_and_shapes_are_unchanged(): - """Whatever the arithmetic inside, the stored arrays stay float32. - - A change to the intermediate precision must not also widen what is - returned, because both arrays are consumed as float32 by the attention - plugin. - """ - num_pos = 256 - inv_freq, table = RopeEmbeddingUtils.create_sinusoidal_positions_for_attention_plugin( - num_pos, HEAD_DIM, THETA - ) - assert inv_freq.dtype == np.float32 - assert table.dtype == np.float32 - assert inv_freq.shape == (HEAD_DIM // 2,) - assert table.shape == (1, num_pos * HEAD_DIM) From 6995aa1acfbfb4c29f590d0475e77962eeba2a5d Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Mon, 14 Sep 2026 04:41:35 +0000 Subject: [PATCH 5/6] Address trivial review comments Signed-off-by: Brian Nguyen --- .../_torch/attention/test_rotary_embedding.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/unittest/_torch/attention/test_rotary_embedding.py b/tests/unittest/_torch/attention/test_rotary_embedding.py index 8e14bff36533..466762a23318 100644 --- a/tests/unittest/_torch/attention/test_rotary_embedding.py +++ b/tests/unittest/_torch/attention/test_rotary_embedding.py @@ -12,6 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import numpy as np import pytest import torch @@ -187,19 +188,26 @@ def test_attention_plugin_duplicate_matches_manual_cat( self, dim, max_positions, theta): """duplicate_data=True should equal creating without duplication then manually concatenating, which is what _normalize_mla_rotary_cache_layout did.""" - _, cs_no = RopeEmbeddingUtils.create_sinusoidal_positions_for_attention_plugin( + inv_freq_no, cs_no = RopeEmbeddingUtils.create_sinusoidal_positions_for_attention_plugin( num_pos=max_positions, dim=dim, theta=theta, duplicate_data=False, ) - _, cs_yes = RopeEmbeddingUtils.create_sinusoidal_positions_for_attention_plugin( + inv_freq_yes, cs_yes = RopeEmbeddingUtils.create_sinusoidal_positions_for_attention_plugin( num_pos=max_positions, dim=dim, theta=theta, duplicate_data=True, ) + # The rotary table dtype is pinned to fp32 (see + # create_sinusoidal_positions_for_attention_plugin); guard that here. + assert inv_freq_no.dtype == np.float32 + assert inv_freq_yes.dtype == np.float32 + assert cs_no.dtype == np.float32 + assert cs_yes.dtype == np.float32 + t = torch.tensor(cs_no).view(max_positions, -1, 2) expected = torch.cat([t, t], dim=1).reshape(1, -1).contiguous() actual = torch.tensor(cs_yes) From be07af57f6960cf688a60ed12c3c1e9f1a585214 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Mon, 14 Sep 2026 14:33:40 -0500 Subject: [PATCH 6/6] [None][chore] Drop the dtype assertions from the rotary-table test The fp32 pin is documented at the variable itself in create_sinusoidal_positions_for_attention_plugin, which is no longer a caller argument, so the test-side guard is redundant (review feedback). Signed-off-by: Brian Nguyen --- .../_torch/attention/test_rotary_embedding.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/tests/unittest/_torch/attention/test_rotary_embedding.py b/tests/unittest/_torch/attention/test_rotary_embedding.py index 466762a23318..8e14bff36533 100644 --- a/tests/unittest/_torch/attention/test_rotary_embedding.py +++ b/tests/unittest/_torch/attention/test_rotary_embedding.py @@ -12,7 +12,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import numpy as np import pytest import torch @@ -188,26 +187,19 @@ def test_attention_plugin_duplicate_matches_manual_cat( self, dim, max_positions, theta): """duplicate_data=True should equal creating without duplication then manually concatenating, which is what _normalize_mla_rotary_cache_layout did.""" - inv_freq_no, cs_no = RopeEmbeddingUtils.create_sinusoidal_positions_for_attention_plugin( + _, cs_no = RopeEmbeddingUtils.create_sinusoidal_positions_for_attention_plugin( num_pos=max_positions, dim=dim, theta=theta, duplicate_data=False, ) - inv_freq_yes, cs_yes = RopeEmbeddingUtils.create_sinusoidal_positions_for_attention_plugin( + _, cs_yes = RopeEmbeddingUtils.create_sinusoidal_positions_for_attention_plugin( num_pos=max_positions, dim=dim, theta=theta, duplicate_data=True, ) - # The rotary table dtype is pinned to fp32 (see - # create_sinusoidal_positions_for_attention_plugin); guard that here. - assert inv_freq_no.dtype == np.float32 - assert inv_freq_yes.dtype == np.float32 - assert cs_no.dtype == np.float32 - assert cs_yes.dtype == np.float32 - t = torch.tensor(cs_no).view(max_positions, -1, 2) expected = torch.cat([t, t], dim=1).reshape(1, -1).contiguous() actual = torch.tensor(cs_yes)