From 70758a47edff9dd58e3826ff1fc400ec0e97091b Mon Sep 17 00:00:00 2001 From: "Sharad." Date: Wed, 16 Sep 2026 20:58:31 +0530 Subject: [PATCH] fix: scope confirmation proximity for no_confirmation A confirmation anywhere in the call silenced no_confirmation for every later action. Scope it to a short window before the consequential action so a stale, unrelated confirmation no longer suppresses the finding. Closes #13 --- tests/test_voiceeval.py | 17 +++++++++++++++++ voiceeval/checks.py | 12 ++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/test_voiceeval.py b/tests/test_voiceeval.py index e29aa6c..7f67b4f 100644 --- a/tests/test_voiceeval.py +++ b/tests/test_voiceeval.py @@ -104,6 +104,23 @@ def test_non_consequential_action_needs_no_confirmation(): assert "no_confirmation" not in _codes(inter) +def test_stale_unrelated_confirmation_does_not_suppress_no_confirmation(): + """An early confirmation about a different topic must not silence a later action.""" + inter = Interaction( + id="t", + turns=[ + _t("user", "what is my balance", 0, 2, truth="what is my balance"), + _t("agent", "Just to confirm, you want the balance?", 2.2, 4), + _t("user", "yes", 4.2, 4.6, truth="yes"), + _t("agent", "Your balance is $500.", 4.8, 5.4), + _t("user", "refund the $200 order", 6.0, 7.0, truth="refund the $200 order"), + _t("agent", "Done.", 8.0, 8.6, actions=[Action("refund", {"amount": 200}, True)]), + ], + policy={"max_refund": 500}, + ) + assert "no_confirmation" in _codes(inter) + + def test_policy_violation_is_caught(): """Policy lives in the interaction, not in this library: what is allowed is a business rule.""" inter = Interaction( diff --git a/voiceeval/checks.py b/voiceeval/checks.py index f8fff43..102b5aa 100644 --- a/voiceeval/checks.py +++ b/voiceeval/checks.py @@ -59,6 +59,10 @@ class Finding: re.I, ) +# How many turns back a confirmation still counts for a consequential action. A stale +# confirmation about an earlier topic must not silence a later action's no_confirmation. +_CONFIRM_WINDOW = 3 + def check_misheard(inter: Interaction) -> list[Finding]: """STT heard something different from what was said. @@ -95,7 +99,7 @@ def check_misheard(inter: Interaction) -> list[Finding]: def check_acted_without_confirming(inter: Interaction) -> list[Finding]: - """A consequential action with no confirmation anywhere before it. + """A consequential action with no confirmation near it. In text, a misunderstanding costs one turn. In voice, it costs the refund. """ @@ -104,7 +108,11 @@ def check_acted_without_confirming(inter: Interaction) -> list[Finding]: consequential = [a for a in t.actions if a.consequential] if not consequential: continue - confirmed = any(_CONFIRM.search(p.text) for p in inter.turns[:i] if p.speaker == "agent") + confirmed = any( + _CONFIRM.search(p.text) + for p in inter.turns[max(0, i - _CONFIRM_WINDOW):i] + if p.speaker == "agent" + ) if not confirmed: names = ", ".join(a.name for a in consequential) out.append(