From c391aec8c6c6bdab22780beb947a6f2d24a6cf4b Mon Sep 17 00:00:00 2001 From: Max Freedom Pollard <272618364+MaxFreedomPollard@users.noreply.github.com> Date: Sun, 6 Sep 2026 11:36:25 -0400 Subject: [PATCH] fix(metrics): mask ignored labels in padding-free seq_acc `compute_acc` in swift/metrics/acc.py scores each packed sequence by slicing `preds` and `labels` with `cu_seqlens`, but that branch never applies the `labels != -100` mask that the padded branch right below it applies. Prompt tokens carry the label -100 and predictions are token ids, so every slice that contains a prompt compares -100 against a token id and `np.all` returns False. With `--padding_free` and `--acc_strategy seq`, seq_acc is therefore reported as 0 for every sequence, including sequences the model answered perfectly. Slice `masks` alongside `preds` and `labels` so the padding-free branch scores the same positions as the padded one. Added tests/utils/test_acc_metrics.py, which checks a packed batch of two sequences and asserts the padding-free result equals the result for the same data as an ordinary padded batch. --- swift/metrics/acc.py | 3 ++- tests/utils/test_acc_metrics.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 tests/utils/test_acc_metrics.py diff --git a/swift/metrics/acc.py b/swift/metrics/acc.py index e513d285bc..98b2b35767 100644 --- a/swift/metrics/acc.py +++ b/swift/metrics/acc.py @@ -34,7 +34,8 @@ def compute_acc(preds, # padding_free for i in range(cu_seqlens.shape[0] - 1): start, end = cu_seqlens[i], cu_seqlens[i + 1] - acc_list.append(np.all(preds[0, start:end] == labels[0, start:end])) + mask = masks[0, start:end] + acc_list.append(np.all(preds[0, start:end][mask] == labels[0, start:end][mask])) else: for i, m in enumerate(masks): acc_list.append(np.all(preds[i, m] == labels[i, m])) diff --git a/tests/utils/test_acc_metrics.py b/tests/utils/test_acc_metrics.py new file mode 100644 index 0000000000..e49a42cb68 --- /dev/null +++ b/tests/utils/test_acc_metrics.py @@ -0,0 +1,32 @@ +import torch +import unittest + +from swift.metrics import compute_acc + +# Two sequences, each two prompt tokens (label -100) followed by two response tokens. +# The prediction at position i is the guess for token i + 1, so the first sequence is +# answered correctly and the second one gets its final token wrong. +PACKED_LABELS = torch.tensor([[-100, -100, 5, 6, -100, -100, 7, 8]]) +PACKED_PREDS = torch.tensor([[0, 5, 6, 0, 0, 7, 9, 0]]) +CU_SEQLENS = torch.tensor([0, 4, 8]) + +BATCH_LABELS = torch.tensor([[-100, -100, 5, 6], [-100, -100, 7, 8]]) +BATCH_PREDS = torch.tensor([[0, 5, 6, 0], [0, 7, 9, 0]]) + + +class TestComputeAcc(unittest.TestCase): + + def test_padding_free_seq_acc_skips_ignored_labels(self): + metrics = compute_acc(PACKED_PREDS, PACKED_LABELS, acc_strategy='seq', cu_seqlens=CU_SEQLENS) + + self.assertEqual([bool(acc) for acc in metrics['seq_acc']], [True, False]) + + def test_padding_free_seq_acc_matches_the_padded_batch(self): + packed = compute_acc(PACKED_PREDS, PACKED_LABELS, acc_strategy='seq', cu_seqlens=CU_SEQLENS) + padded = compute_acc(BATCH_PREDS, BATCH_LABELS, acc_strategy='seq') + + self.assertEqual([bool(acc) for acc in packed['seq_acc']], [bool(acc) for acc in padded['seq_acc']]) + + +if __name__ == '__main__': + unittest.main()