From 0927b41026e7d822ac2fa0db82e1bb3df8d2dbda Mon Sep 17 00:00:00 2001 From: Joey Kudish Date: Sat, 8 Aug 2026 13:28:45 -0700 Subject: [PATCH] feat: adds scorer result callbacks --- src/Configuration.php | 25 ++++ src/Events/Scored.php | 24 ++++ src/Scorers/ScorerAssertion.php | 15 ++- tests/Unit/ScoredHookTest.php | 201 ++++++++++++++++++++++++++++++++ 4 files changed, 263 insertions(+), 2 deletions(-) create mode 100644 src/Events/Scored.php create mode 100644 tests/Unit/ScoredHookTest.php diff --git a/src/Configuration.php b/src/Configuration.php index 3c38105..09246fa 100644 --- a/src/Configuration.php +++ b/src/Configuration.php @@ -11,6 +11,7 @@ use Pest\Evals\Drivers\ClosureJudge; use Pest\Evals\Drivers\LaravelAiEmbeddings; use Pest\Evals\Drivers\LaravelAiJudge; +use Pest\Evals\Events\Scored; final class Configuration { @@ -18,6 +19,11 @@ final class Configuration private static ?EmbeddingsDriver $embeddings = null; + /** + * @var array + */ + private static array $afterScoredCallbacks = []; + public static function resolvedJudge(): JudgeDriver { return self::$judge ?? new LaravelAiJudge(); @@ -42,6 +48,15 @@ public static function flush(): void { self::$judge = null; self::$embeddings = null; + self::$afterScoredCallbacks = []; + } + + /** @internal */ + public static function dispatchScored(Scored $event): void + { + foreach (self::$afterScoredCallbacks as $callback) { + $callback($event); + } } public function judgeUsing(JudgeDriver|Closure $judge): self @@ -57,4 +72,14 @@ public function embeddingsUsing(EmbeddingsDriver|Closure $embeddings): self return $this; } + + /** + * @param Closure(Scored): void $callback + */ + public function afterScored(Closure $callback): self + { + self::$afterScoredCallbacks[] = $callback; + + return $this; + } } diff --git a/src/Events/Scored.php b/src/Events/Scored.php new file mode 100644 index 0000000..88e0d4a --- /dev/null +++ b/src/Events/Scored.php @@ -0,0 +1,24 @@ +passed = $result->passed($threshold); + } +} diff --git a/src/Scorers/ScorerAssertion.php b/src/Scorers/ScorerAssertion.php index 1d42107..3637a87 100644 --- a/src/Scorers/ScorerAssertion.php +++ b/src/Scorers/ScorerAssertion.php @@ -9,6 +9,7 @@ use Pest\Evals\Contracts\RequiresEmbeddings; use Pest\Evals\Contracts\RequiresJudge; use Pest\Evals\Eval\Context; +use Pest\Evals\Events\Scored; use Pest\Evals\Plugin; use Pest\Evals\Support\VerbosePanel; @@ -43,13 +44,23 @@ public function assert(Scorer $scorer, array $outputs, float $threshold, ?Contex $result = $scorer->score($input, $sampleOutput, $expected); $scorerName = class_basename($result->scorer); - $passed = $result->score >= $threshold; + $event = new Scored( + result: $result, + threshold: $threshold, + input: $input, + output: $sampleOutput, + expected: $expected, + sample: $index + 1, + samples: $samples, + ); + + Configuration::dispatchScored($event); if (Plugin::isVerbose()) { $this->panel->render( scorer: $scorerName, threshold: $threshold, - passed: $passed, + passed: $event->passed, input: $input, output: $sampleOutput, reasoning: $result->reasoning, diff --git a/tests/Unit/ScoredHookTest.php b/tests/Unit/ScoredHookTest.php new file mode 100644 index 0000000..676f922 --- /dev/null +++ b/tests/Unit/ScoredHookTest.php @@ -0,0 +1,201 @@ +score, $this->reasoning, self::class); + } + }; +} + +it('reports a passing custom scorer result with complete context', function (): void { + $events = []; + + pest()->evals()->afterScored(function (Scored $event) use (&$events): void { + $events[] = $event; + }); + + (new ScorerAssertion)->assert( + scorer: scorerReturning(0.91), + outputs: ['Paris'], + threshold: 0.8, + expected: 'Paris', + ); + + expect($events)->toHaveCount(1) + ->and($events[0]->result->score)->toBe(0.91) + ->and($events[0]->threshold)->toBe(0.8) + ->and($events[0]->input)->toBe('') + ->and($events[0]->output)->toBe('Paris') + ->and($events[0]->expected)->toBe('Paris') + ->and($events[0]->sample)->toBe(1) + ->and($events[0]->samples)->toBe(1) + ->and($events[0]->passed)->toBeTrue(); +}); + +it('reports a score equal to the threshold as passing', function (): void { + $events = []; + + pest()->evals()->afterScored(function (Scored $event) use (&$events): void { + $events[] = $event; + }); + + (new ScorerAssertion)->assert(scorerReturning(0.8), ['output'], 0.8); + + expect($events)->toHaveCount(1) + ->and($events[0]->passed)->toBeTrue(); +}); + +it('reports a failing result before the threshold assertion fails', function (): void { + $events = []; + + pest()->evals()->afterScored(function (Scored $event) use (&$events): void { + $events[] = $event; + }); + + expect(fn () => (new ScorerAssertion)->assert( + scorer: scorerReturning(0.2, 'not relevant'), + outputs: ['Madrid'], + threshold: 0.7, + ))->toThrow(ExpectationFailedException::class); + + expect($events)->toHaveCount(1) + ->and($events[0]->passed)->toBeFalse() + ->and($events[0]->result->reasoning)->toBe('not relevant'); +}); + +it('reports every repeated sample in order through the public expectation API', function (): void { + $events = []; + $output = 0; + + pest()->evals()->afterScored(function (Scored $event) use (&$events): void { + $events[] = $event; + }); + + $_SERVER['PEST_EVALS'] = '1'; + + expect(function (string $prompt) use (&$output): string { + $output++; + + return "{$prompt} {$output}"; + })->prompt('sample')->repeat(3)->toPassScorer(scorerReturning(1.0)); + + expect(array_map(fn (Scored $event): array => [ + $event->input, + $event->output, + $event->sample, + $event->samples, + ], $events))->toBe([ + ['sample', 'sample 1', 1, 3], + ['sample', 'sample 2', 2, 3], + ['sample', 'sample 3', 3, 3], + ]); +}); + +it('reports built-in scorer results through the public expectation API', function (): void { + $events = []; + + pest()->evals() + ->judgeUsing(fn (): string => '{"score":0.88,"reasoning":"direct answer"}') + ->afterScored(function (Scored $event) use (&$events): void { + $events[] = $event; + }); + + expect('Four')->toBeRelevant(0.8); + + expect($events)->toHaveCount(1) + ->and($events[0]->result->scorer)->toBe(Relevance::class) + ->and($events[0]->result->score)->toBe(0.88) + ->and($events[0]->passed)->toBeTrue(); +}); + +it('does not report when scoring is disabled', function (): void { + $events = []; + + pest()->evals()->afterScored(function (Scored $event) use (&$events): void { + $events[] = $event; + }); + + $scorer = new class implements RequiresJudge, Scorer + { + public function score(string $input, string $output, ?string $expected = null): ScorerResult + { + throw new RuntimeException('The scorer must not run.'); + } + }; + + (new ScorerAssertion)->assert($scorer, ['not scored'], 0.7); + + expect($events)->toBe([]); +}); + +it('propagates callback exceptions before the threshold assertion', function (): void { + pest()->evals()->afterScored(function (): never { + throw new RuntimeException('Recorder unavailable.'); + }); + + expect(fn () => (new ScorerAssertion)->assert( + scorer: scorerReturning(0.1), + outputs: ['failing output'], + threshold: 0.9, + ))->toThrow(RuntimeException::class, 'Recorder unavailable.'); +}); + +it('notifies multiple callbacks in registration order', function (): void { + $calls = []; + + pest()->evals()->afterScored(function () use (&$calls): void { + $calls[] = 'first'; + }); + + pest()->evals()->afterScored(function () use (&$calls): void { + $calls[] = 'second'; + }); + + (new ScorerAssertion)->assert(scorerReturning(1.0), ['output'], 0.7); + + expect($calls)->toBe(['first', 'second']); +}); + +it('clears registered callbacks when the configuration is flushed', function (): void { + $calls = 0; + + pest()->evals()->afterScored(function () use (&$calls): void { + $calls++; + }); + + Configuration::flush(); + + (new ScorerAssertion)->assert(scorerReturning(1.0), ['output'], 0.7); + + expect($calls)->toBe(0); +});