DevserverTest.java

package dev.braintrust.devserver;

import static dev.braintrust.json.BraintrustJsonMapper.fromJson;
import static org.junit.jupiter.api.Assertions.*;

import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import dev.braintrust.BraintrustUtils;
import dev.braintrust.TestHarness;
import dev.braintrust.eval.Score;
import dev.braintrust.eval.Scorer;
import io.opentelemetry.api.common.AttributeKey;
import io.opentelemetry.api.trace.StatusCode;
import io.opentelemetry.sdk.trace.data.SpanData;
import java.io.BufferedReader;
import java.io.InputStreamReader;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.junit.jupiter.api.*;

@Slf4j
class DevserverTest {
    private static Devserver server;
    private static Thread serverThread;
    private static TestHarness testHarness;
    // private static final int TEST_PORT = TestUtils.getRandomOpenPort();
    private static final int TEST_PORT = 8301;
    private static final String TEST_URL = "http://localhost:" + TEST_PORT;
    private static final ObjectMapper JSON_MAPPER = new ObjectMapper();

    private static final String REMOTE_EVAL_NAME = "food-type-classifier";
    private static final String PARAM_EVAL_NAME = "param-eval";
    private static final String TASK_ERROR_EVAL_NAME = "task-error-eval";
    private static final String SCORER_ERROR_EVAL_NAME = "scorer-error-eval";
    private static final BraintrustUtils.Parent PLAYGROUND_PARENT =
            new BraintrustUtils.Parent("playground_id", "ceea7422-3507-4d1c-a5f7-7acf41d9fac2");

    // Remote scorer created in the java-unit-test project via the test harness. Its latest version
    // returns 1.0 for an exact match and 0.0 otherwise. The score is keyed in results by the
    // scorer's resolved name: "invoke-<project>-<slug>-<version>".
    private static final String REMOTE_SCORER_SLUG = "typescript-exact-match";
    private static String remoteScorerFunctionId;
    // Resolved scorer name (set in setUp once the project name is known).
    private static String REMOTE_SCORER_NAME;
    // The name embedded in the remote scorer's return payload (see REMOTE_SCORER_CODE). When the
    // remote invoke actually executes (VCR record/off), the score is keyed by this name. In replay
    // mode the invoke request cannot be matched by a cassette (its body embeds live span IDs via
    // `parent`), so the scorer falls back to scoreForScorerException and the score is keyed by
    // REMOTE_SCORER_NAME instead. Assertions must accept both.
    private static final String REMOTE_SCORER_PAYLOAD_NAME = "typescript exact match";

    @BeforeAll
    static void setUp() throws Exception {
        // Set up test harness with VCR (records/replays HTTP interactions)
        testHarness = TestHarness.setup();

        // Ensure the remote code scorer exists and resolve its function ID. The latest version
        // returns {name: "typescript exact match", score: output === expected ? 1.0 : 0.0}.
        var scorerInfo = testHarness.ensureRemoteCodeScorer(REMOTE_SCORER_SLUG, REMOTE_SCORER_CODE);
        remoteScorerFunctionId =
                lookupFunctionId(TestHarness.defaultProjectName(), scorerInfo.slug());
        REMOTE_SCORER_NAME =
                "invoke-" + TestHarness.defaultProjectName() + "-" + scorerInfo.slug() + "-latest";

        // Create a shared eval for all tests
        RemoteEval<String, String> testEval =
                RemoteEval.<String, String>builder()
                        .name(REMOTE_EVAL_NAME)
                        .taskFunction(
                                input -> {
                                    // Create a span inside the task to test baggage propagation
                                    var tracer = dev.braintrust.trace.BraintrustTracing.getTracer();
                                    var span = tracer.spanBuilder("custom-task-span").startSpan();
                                    try (var scope =
                                            io.opentelemetry.context.Context.current()
                                                    .with(span)
                                                    .makeCurrent()) {
                                        // Do some work
                                        return "java-fruit";
                                    } finally {
                                        span.end();
                                    }
                                })
                        .scorer(Scorer.of("simple_scorer", (expected, result) -> 0.7))
                        .build();

        // Eval whose task throws for "bad-input"
        RemoteEval<String, String> taskErrorEval =
                RemoteEval.<String, String>builder()
                        .name(TASK_ERROR_EVAL_NAME)
                        .taskFunction(
                                input -> {
                                    if ("bad-input".equals(input)) {
                                        throw new RuntimeException("task failed on bad-input");
                                    }
                                    return "result";
                                })
                        .scorer(Scorer.of("exact_match", (expected, result) -> 0.7))
                        .build();

        // Eval with a scorer that always throws
        RemoteEval<String, String> scorerErrorEval =
                RemoteEval.<String, String>builder()
                        .name(SCORER_ERROR_EVAL_NAME)
                        .taskFunction(input -> "result")
                        .scorer(
                                new Scorer<String, String>() {
                                    @Override
                                    public String getName() {
                                        return "broken_scorer";
                                    }

                                    @Override
                                    public List<Score> score(
                                            dev.braintrust.eval.TaskResult<String, String>
                                                    taskResult) {
                                        throw new RuntimeException("scorer is broken");
                                    }
                                })
                        .scorer(Scorer.of("working_scorer", (expected, result) -> 1.0))
                        .build();

        // Eval with parameters — task returns the model param as its output
        RemoteEval<String, String> paramEval =
                RemoteEval.<String, String>builder()
                        .name(PARAM_EVAL_NAME)
                        .parameter(dev.braintrust.eval.ParameterDef.data("model", "gpt-4"))
                        .parameter(dev.braintrust.eval.ParameterDef.data("temperature", 0.5))
                        .task(
                                new dev.braintrust.eval.Task<>() {
                                    @Override
                                    public dev.braintrust.eval.TaskResult<String, String> apply(
                                            dev.braintrust.eval.DatasetCase<String, String>
                                                    datasetCase,
                                            dev.braintrust.eval.Parameters parameters)
                                            throws Exception {
                                        // Echo both params so tests can verify defaults + overrides
                                        String model = parameters.get("model", String.class);
                                        Double temp = parameters.get("temperature", Double.class);
                                        String output = model + ":" + temp;
                                        return new dev.braintrust.eval.TaskResult<>(
                                                output, datasetCase, parameters);
                                    }
                                })
                        .scorer(Scorer.of("static_scorer", (expected, result) -> 1.0))
                        .build();

        server =
                Devserver.builder()
                        .config(testHarness.braintrust().config())
                        .registerEval(testEval)
                        .registerEval(paramEval)
                        .registerEval(taskErrorEval)
                        .registerEval(scorerErrorEval)
                        .host("localhost")
                        .port(TEST_PORT)
                        .build();

        // Start server in background thread
        serverThread =
                new Thread(
                        () -> {
                            try {
                                server.start();
                            } catch (Exception e) {
                                log.error("unable to start dev server", e);
                            }
                        });
        serverThread.start();

        // Give server time to start
        Thread.sleep(1000);
    }

    @AfterAll
    @SneakyThrows
    static void tearDown() {
        if (server != null) {
            server.stop();
        }
        if (serverThread != null) {
            serverThread.join(30_000);
            if (serverThread.isAlive()) {
                serverThread.interrupt();
            }
        }
    }

    @Test
    void testHealthCheck() throws Exception {
        // Test health check endpoint using the shared devserver
        HttpClient client = HttpClient.newHttpClient();
        HttpRequest request =
                HttpRequest.newBuilder().uri(URI.create(TEST_URL + "/")).GET().build();

        HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

        assertEquals(200, response.statusCode());
        assertEquals("Hello, world!", response.body());
        assertEquals("text/plain", response.headers().firstValue("Content-Type").orElse(""));
    }

    @Test
    void testStreamingEval() throws Exception {
        // Create eval request with inline data using EvalRequest types
        EvalRequest evalRequest = new EvalRequest();
        evalRequest.setName(REMOTE_EVAL_NAME);
        evalRequest.setStream(true);

        // Create inline data
        EvalRequest.DataSpec dataSpec = new EvalRequest.DataSpec();

        EvalRequest.EvalCaseData case1 = new EvalRequest.EvalCaseData();
        case1.setInput("apple");
        case1.setExpected("fruit");

        EvalRequest.EvalCaseData case2 = new EvalRequest.EvalCaseData();
        case2.setInput("carrot");
        case2.setExpected("vegetable");

        dataSpec.setData(List.of(case1, case2));
        evalRequest.setData(dataSpec);

        Map<String, Object> parentSpec =
                Map.of(
                        "object_type", PLAYGROUND_PARENT.type(),
                        "object_id", PLAYGROUND_PARENT.id(),
                        "propagated_event",
                                Map.of("span_attributes", Map.of("generation", "test-gen-1")));
        evalRequest.setParent(parentSpec);

        // Add remote scorer from Braintrust
        EvalRequest.RemoteScorer remoteScorer = new EvalRequest.RemoteScorer();
        remoteScorer.setName(REMOTE_SCORER_NAME);
        EvalRequest.FunctionId functionId = new EvalRequest.FunctionId();
        functionId.setFunctionId(remoteScorerFunctionId);
        remoteScorer.setFunctionId(functionId);
        evalRequest.setScores(List.of(remoteScorer));

        String requestBody = JSON_MAPPER.writeValueAsString(evalRequest);

        // Make POST request to /eval with auth headers
        HttpURLConnection conn =
                (HttpURLConnection) new URI(TEST_URL + "/eval").toURL().openConnection();
        conn.setRequestMethod("POST");
        conn.setRequestProperty("Content-Type", "application/json");
        conn.setRequestProperty("x-bt-auth-token", testHarness.braintrustApiKey());
        conn.setRequestProperty("x-bt-project-id", TestHarness.defaultProjectId());
        conn.setRequestProperty("x-bt-org-name", TestHarness.defaultOrgName());
        conn.setDoOutput(true);

        // Write request body
        conn.getOutputStream().write(requestBody.getBytes(StandardCharsets.UTF_8));
        conn.getOutputStream().flush();

        // Read SSE response
        assertEquals(200, conn.getResponseCode());
        assertEquals("text/event-stream", conn.getHeaderField("Content-Type"));

        BufferedReader reader =
                new BufferedReader(
                        new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8));

        List<Map<String, String>> events = new ArrayList<>();
        String line;
        String currentEvent = null;
        StringBuilder currentData = new StringBuilder();

        while ((line = reader.readLine()) != null) {
            if (line.startsWith("event: ")) {
                currentEvent = line.substring(7);
            } else if (line.startsWith("data: ")) {
                currentData.append(line.substring(6));
            } else if (line.isEmpty() && currentEvent != null) {
                // End of event
                events.add(Map.of("event", currentEvent, "data", currentData.toString()));
                currentEvent = null;
                currentData = new StringBuilder();
            }
        }
        reader.close();

        // Assert event structure
        assertFalse(events.isEmpty(), "Should have received events");

        // Count events by type
        List<Map<String, String>> progressEvents =
                events.stream().filter(e -> "progress".equals(e.get("event"))).toList();
        List<Map<String, String>> summaryEvents =
                events.stream().filter(e -> "summary".equals(e.get("event"))).toList();
        List<Map<String, String>> doneEvents =
                events.stream().filter(e -> "done".equals(e.get("event"))).toList();

        // Should have 1 start event, 2 progress events (one per dataset case), 1 summary, 1 done
        assertEquals(2, progressEvents.size(), "Should have 2 progress events");
        assertEquals(1, summaryEvents.size(), "Should have 1 summary event");
        assertEquals(1, doneEvents.size(), "Should have 1 done event");

        // Verify progress events match expected structure
        for (Map<String, String> progressEvent : progressEvents) {
            String dataJson = progressEvent.get("data");
            JsonNode progressData = JSON_MAPPER.readTree(dataJson);

            // Assert expected fields in progress event
            assertTrue(progressData.has("id"), "Progress event should have id");
            assertEquals("task", progressData.get("object_type").asText());
            assertEquals(REMOTE_EVAL_NAME, progressData.get("name").asText());
            assertEquals("code", progressData.get("format").asText());
            assertEquals("completion", progressData.get("output_type").asText());
            assertEquals("json_delta", progressData.get("event").asText());

            // Assert data field contains the task result
            String taskResultJson = progressData.get("data").asText();
            assertEquals("\"java-fruit\"", taskResultJson);
        }

        { // Verify summary event
            Map<String, String> summaryEvent = summaryEvents.get(0);
            JsonNode summaryData = JSON_MAPPER.readTree(summaryEvent.get("data"));

            assertEquals(TestHarness.defaultProjectName(), summaryData.get("projectName").asText());
            assertTrue(summaryData.has("projectId"));
            assertEquals(REMOTE_EVAL_NAME, summaryData.get("experimentName").asText());

            // Verify scores in summary
            assertTrue(summaryData.has("scores"));
            JsonNode scores = summaryData.get("scores");

            // Verify local scorer
            assertTrue(scores.has("simple_scorer"), "Summary should have simple_scorer");
            JsonNode simpleScorer = scores.get("simple_scorer");
            assertEquals("simple_scorer", simpleScorer.get("name").asText());
            assertEquals(0.7, simpleScorer.get("score").asDouble(), 0.001);

            // Verify remote scorer (returns 0.0 because output "java-fruit" != expected
            // "fruit"/"vegetable"). Keyed by the payload name on a real invoke (record/off) or
            // by the resolved scorer name when replay falls back on an unmatched invoke request.
            String remoteScorerKey =
                    scores.has(REMOTE_SCORER_PAYLOAD_NAME)
                            ? REMOTE_SCORER_PAYLOAD_NAME
                            : REMOTE_SCORER_NAME;
            assertTrue(
                    scores.has(remoteScorerKey),
                    "Summary should have remote scorer under '%s' or '%s' -- got: %s"
                            .formatted(REMOTE_SCORER_PAYLOAD_NAME, REMOTE_SCORER_NAME, scores));
            JsonNode remoteScorerResult = scores.get(remoteScorerKey);
            assertEquals(remoteScorerKey, remoteScorerResult.get("name").asText());
            assertEquals(0.0, remoteScorerResult.get("score").asDouble(), 0.001);
        }

        // Get exported spans from test harness (since devserver uses global tracer)
        // Filter to only spans belonging to this test by finding trace IDs from eval spans
        // with the right generation tag
        List<SpanData> allSpans = testHarness.awaitExportedSpans();
        var traceIds =
                allSpans.stream()
                        .filter(
                                s -> {
                                    var attrs =
                                            s.getAttributes()
                                                    .get(
                                                            AttributeKey.stringKey(
                                                                    "braintrust.span_attributes"));
                                    return attrs != null && attrs.contains("test-gen-1");
                                })
                        .map(s -> s.getTraceId())
                        .collect(java.util.stream.Collectors.toSet());
        List<SpanData> exportedSpans =
                allSpans.stream().filter(s -> traceIds.contains(s.getTraceId())).toList();
        assertFalse(exportedSpans.isEmpty(), "Should have exported spans");

        // We should have 2 eval traces (one per dataset case), each with task, scores, and custom
        // spans
        // Each trace has: 1 eval span, 1 task span, 2 score spans (local + remote), 1
        // custom-task-span = 5 spans
        // per case
        // Total: 2 cases * 5 spans = 10 spans
        assertEquals(10, exportedSpans.size(), "Should have 10 spans (5 per dataset case)");

        // Verify span types
        var evalSpans = exportedSpans.stream().filter(s -> s.getName().equals("eval")).toList();
        var taskSpans = exportedSpans.stream().filter(s -> s.getName().equals("task")).toList();
        var scoreSpans = exportedSpans.stream().filter(s -> s.getName().equals("score")).toList();
        var customSpans =
                exportedSpans.stream().filter(s -> s.getName().equals("custom-task-span")).toList();

        assertEquals(2, evalSpans.size(), "Should have 2 eval spans");
        assertEquals(2, taskSpans.size(), "Should have 2 task spans");
        assertEquals(4, scoreSpans.size(), "Should have 4 score spans (2 scorers x 2 cases)");
        assertEquals(2, customSpans.size(), "Should have 2 custom-task-span spans");

        // Verify eval spans have all required attributes
        for (SpanData evalSpan : evalSpans) {
            // Verify braintrust.parent
            String parent =
                    evalSpan.getAttributes()
                            .get(
                                    io.opentelemetry.api.common.AttributeKey.stringKey(
                                            "braintrust.parent"));
            assertEquals(
                    PLAYGROUND_PARENT.toParentValue(),
                    parent,
                    "Eval span should have parent attribute");

            String spanAttrsJson =
                    evalSpan.getAttributes()
                            .get(
                                    io.opentelemetry.api.common.AttributeKey.stringKey(
                                            "braintrust.span_attributes"));
            assertNotNull(spanAttrsJson, "Eval span should have span_attributes");
            JsonNode spanAttrs = JSON_MAPPER.readTree(spanAttrsJson);
            assertEquals("eval", spanAttrs.get("type").asText());
            assertEquals("eval", spanAttrs.get("name").asText());
            assertEquals("test-gen-1", spanAttrs.get("generation").asText());

            String inputJson =
                    evalSpan.getAttributes()
                            .get(
                                    io.opentelemetry.api.common.AttributeKey.stringKey(
                                            "braintrust.input_json"));
            assertNotNull(inputJson, "Eval span should have input_json");

            String expectedJson =
                    evalSpan.getAttributes()
                            .get(
                                    io.opentelemetry.api.common.AttributeKey.stringKey(
                                            "braintrust.expected_json"));
            assertNotNull(expectedJson, "Eval span should have expected_json");

            String outputJson =
                    evalSpan.getAttributes()
                            .get(
                                    io.opentelemetry.api.common.AttributeKey.stringKey(
                                            "braintrust.output_json"));
            assertNotNull(outputJson, "Eval span should have output_json");
            JsonNode output = JSON_MAPPER.readTree(outputJson);
            assertEquals("java-fruit", output.get("output").asText());
        }

        for (SpanData taskSpan : taskSpans) {
            // Verify braintrust.parent
            String parent =
                    taskSpan.getAttributes()
                            .get(
                                    io.opentelemetry.api.common.AttributeKey.stringKey(
                                            "braintrust.parent"));
            assertEquals(PLAYGROUND_PARENT.toParentValue(), parent);

            String spanAttrsJson =
                    taskSpan.getAttributes()
                            .get(
                                    io.opentelemetry.api.common.AttributeKey.stringKey(
                                            "braintrust.span_attributes"));
            assertNotNull(spanAttrsJson, "Task span should have span_attributes");
            JsonNode spanAttrs = JSON_MAPPER.readTree(spanAttrsJson);
            assertEquals("task", spanAttrs.get("type").asText());
            assertEquals("task", spanAttrs.get("name").asText());
            assertEquals("test-gen-1", spanAttrs.get("generation").asText());

            String inputJson =
                    taskSpan.getAttributes()
                            .get(
                                    io.opentelemetry.api.common.AttributeKey.stringKey(
                                            "braintrust.input_json"));
            assertNotNull(inputJson, "Task span should have input_json");

            String outputJson =
                    taskSpan.getAttributes()
                            .get(
                                    io.opentelemetry.api.common.AttributeKey.stringKey(
                                            "braintrust.output_json"));
            assertNotNull(outputJson, "Task span should have output_json");
            JsonNode output = JSON_MAPPER.readTree(outputJson);
            assertEquals("java-fruit", output.get("output").asText());
        }

        for (SpanData scoreSpan : scoreSpans) {
            // Verify braintrust.parent
            String parent =
                    scoreSpan
                            .getAttributes()
                            .get(
                                    io.opentelemetry.api.common.AttributeKey.stringKey(
                                            "braintrust.parent"));
            assertEquals(PLAYGROUND_PARENT.toParentValue(), parent);

            // Verify braintrust.span_attributes
            String spanAttrsJson =
                    scoreSpan
                            .getAttributes()
                            .get(
                                    io.opentelemetry.api.common.AttributeKey.stringKey(
                                            "braintrust.span_attributes"));
            assertNotNull(spanAttrsJson, "Score span should have span_attributes");
            JsonNode spanAttrs = JSON_MAPPER.readTree(spanAttrsJson);
            assertEquals("score", spanAttrs.get("type").asText());
            assertEquals("scorer", spanAttrs.get("purpose").asText());
            assertEquals("test-gen-1", spanAttrs.get("generation").asText());

            // Scorer name should be either simple_scorer or the remote scorer. The remote
            // scorer's span name is "invoke-<project>-<slug>-<version>".
            String scorerName = spanAttrs.get("name").asText();
            assertTrue(
                    scorerName.contains("simple_scorer") || scorerName.contains(REMOTE_SCORER_SLUG),
                    "Score span name should be simple_scorer or contain %s -- got: %s"
                            .formatted(REMOTE_SCORER_SLUG, scorerName));

            // Verify braintrust.output_json contains scores
            String outputJson =
                    scoreSpan
                            .getAttributes()
                            .get(
                                    io.opentelemetry.api.common.AttributeKey.stringKey(
                                            "braintrust.output_json"));
            assertNotNull(outputJson, "Score span should have output_json");
            JsonNode output = JSON_MAPPER.readTree(outputJson);

            if (scorerName.equals("simple_scorer")) {
                assertTrue(
                        output.has("simple_scorer"), "Output should contain simple_scorer results");
                assertEquals(0.7, output.get("simple_scorer").asDouble(), 0.001);
            } else {
                // Keyed by the payload name on a real invoke (record/off) or by the resolved
                // scorer name when replay falls back on an unmatched invoke request.
                String remoteScorerKey =
                        output.has(REMOTE_SCORER_PAYLOAD_NAME)
                                ? REMOTE_SCORER_PAYLOAD_NAME
                                : REMOTE_SCORER_NAME;
                assertTrue(
                        output.has(remoteScorerKey),
                        "Output should contain remote scorer results under '%s' or '%s' -- got: %s"
                                .formatted(REMOTE_SCORER_PAYLOAD_NAME, REMOTE_SCORER_NAME, output));
                assertEquals(0.0, output.get(remoteScorerKey).asDouble(), 0.001);
            }
        }

        for (SpanData customSpan : customSpans) {
            // Verify it has a parent span (is not a root span)
            assertTrue(
                    customSpan.getParentSpanContext().isValid(),
                    "Custom span should have a valid parent span context");
            assertNotEquals(
                    io.opentelemetry.api.trace.SpanId.getInvalid(),
                    customSpan.getParentSpanId(),
                    "Custom span should not be a root span (should have parent span ID)");

            // Verify it has braintrust.parent attribute from baggage propagation
            String parent =
                    customSpan
                            .getAttributes()
                            .get(
                                    io.opentelemetry.api.common.AttributeKey.stringKey(
                                            "braintrust.parent"));
            assertEquals(PLAYGROUND_PARENT.toParentValue(), parent);
        }
    }

    @Test
    void testExperimentEval() throws Exception {
        // A remote eval triggered as an *Experiment* (snapshot) from the UI sends no playground
        // parent — instead an experiment_name (+ project via headers). The devserver should build
        // and run a standard Eval, which creates a fresh experiment and parents spans to
        // experiment_id:<id>. It then streams only summary + done (no per-case progress).
        final String experimentName = "java-experiment-repro";

        EvalRequest evalRequest = new EvalRequest();
        evalRequest.setName(REMOTE_EVAL_NAME);
        evalRequest.setStream(true);
        evalRequest.setExperimentName(experimentName);
        // Note: no parent set -> experiment-snapshot path.

        EvalRequest.DataSpec dataSpec = new EvalRequest.DataSpec();
        EvalRequest.EvalCaseData case1 = new EvalRequest.EvalCaseData();
        case1.setInput("apple");
        case1.setExpected("fruit");
        EvalRequest.EvalCaseData case2 = new EvalRequest.EvalCaseData();
        case2.setInput("carrot");
        case2.setExpected("vegetable");
        dataSpec.setData(List.of(case1, case2));
        evalRequest.setData(dataSpec);

        String requestBody = JSON_MAPPER.writeValueAsString(evalRequest);

        HttpURLConnection conn =
                (HttpURLConnection) new URI(TEST_URL + "/eval").toURL().openConnection();
        conn.setRequestMethod("POST");
        conn.setRequestProperty("Content-Type", "application/json");
        conn.setRequestProperty("x-bt-auth-token", testHarness.braintrustApiKey());
        conn.setRequestProperty("x-bt-project-id", TestHarness.defaultProjectId());
        conn.setRequestProperty("x-bt-org-name", TestHarness.defaultOrgName());
        conn.setDoOutput(true);
        conn.getOutputStream().write(requestBody.getBytes(StandardCharsets.UTF_8));
        conn.getOutputStream().flush();

        assertEquals(200, conn.getResponseCode());

        List<Map<String, String>> events = readSSEEvents(conn);

        List<Map<String, String>> progressEvents =
                events.stream().filter(e -> "progress".equals(e.get("event"))).toList();
        List<Map<String, String>> summaryEvents =
                events.stream().filter(e -> "summary".equals(e.get("event"))).toList();
        List<Map<String, String>> doneEvents =
                events.stream().filter(e -> "done".equals(e.get("event"))).toList();
        // Snapshots don't stream per-case progress: only a terminating summary + done.
        assertEquals(0, progressEvents.size(), "Snapshot runs should not send progress events");
        assertEquals(1, summaryEvents.size(), "Should have 1 summary event");
        assertEquals(1, doneEvents.size(), "Should have 1 done event");

        // The summary should reference the created experiment (unlike playground runs). The id is
        // whatever the backend assigned — derive it from the summary rather than hardcoding.
        JsonNode summaryData = JSON_MAPPER.readTree(summaryEvents.get(0).get("data"));
        assertEquals(TestHarness.defaultProjectName(), summaryData.get("projectName").asText());
        assertTrue(summaryData.get("experimentName").asText().startsWith(experimentName));
        assertFalse(
                summaryData.get("experimentId").isNull(),
                "experiment run should have an experimentId");
        assertFalse(
                summaryData.get("experimentUrl").isNull(),
                "experiment run should have an experimentUrl");
        String experimentId = summaryData.get("experimentId").asText();

        // Spans should be parented to experiment_id:<id> using the standard Eval span shape.
        List<SpanData> allSpans = testHarness.awaitExportedSpans();
        String expectedParent = "experiment_id:" + experimentId;
        var evalSpans =
                allSpans.stream()
                        .filter(s -> s.getName().equals("eval"))
                        .filter(
                                s ->
                                        expectedParent.equals(
                                                s.getAttributes()
                                                        .get(
                                                                AttributeKey.stringKey(
                                                                        "braintrust.parent"))))
                        .toList();
        assertEquals(2, evalSpans.size(), "Should have 2 eval spans parented to the experiment");

        for (SpanData evalSpan : evalSpans) {
            JsonNode spanAttrs =
                    JSON_MAPPER.readTree(
                            evalSpan.getAttributes()
                                    .get(AttributeKey.stringKey("braintrust.span_attributes")));
            assertEquals("eval", spanAttrs.get("type").asText());
            // Standard Eval span shape uses braintrust.expected (not the playground's
            // expected_json) and carries no playground-specific "generation" key.
            assertFalse(
                    spanAttrs.has("generation"), "experiment spans should not carry generation");
            assertNotNull(
                    evalSpan.getAttributes().get(AttributeKey.stringKey("braintrust.expected")),
                    "standard Eval decorator should set braintrust.expected");
        }
    }

    @Test
    void testMalformedParent() throws Exception {
        // A parent with only one of object_type/object_id set is neither a valid playground run
        // nor an experiment snapshot (which sends no parent at all). extractPlaygroundParent
        // rejects it with an IllegalArgumentException, which is streamed back as an SSE error
        // event.
        EvalRequest evalRequest = new EvalRequest();
        evalRequest.setName(REMOTE_EVAL_NAME);
        evalRequest.setStream(true);

        EvalRequest.DataSpec dataSpec = new EvalRequest.DataSpec();
        EvalRequest.EvalCaseData case1 = new EvalRequest.EvalCaseData();
        case1.setInput("apple");
        case1.setExpected("fruit");
        dataSpec.setData(List.of(case1));
        evalRequest.setData(dataSpec);

        // object_type present but object_id missing -> malformed parent.
        Map<String, Object> parentSpec = Map.of("object_type", PLAYGROUND_PARENT.type());
        evalRequest.setParent(parentSpec);

        String requestBody = JSON_MAPPER.writeValueAsString(evalRequest);

        HttpURLConnection conn =
                (HttpURLConnection) new URI(TEST_URL + "/eval").toURL().openConnection();
        conn.setRequestMethod("POST");
        conn.setRequestProperty("Content-Type", "application/json");
        conn.setRequestProperty("x-bt-auth-token", testHarness.braintrustApiKey());
        conn.setRequestProperty("x-bt-project-id", TestHarness.defaultProjectId());
        conn.setRequestProperty("x-bt-org-name", TestHarness.defaultOrgName());
        conn.setDoOutput(true);
        conn.getOutputStream().write(requestBody.getBytes(StandardCharsets.UTF_8));
        conn.getOutputStream().flush();

        // Streaming has already begun, so the failure surfaces as an SSE error event (not an HTTP
        // error status).
        assertEquals(200, conn.getResponseCode());

        List<Map<String, String>> events = readSSEEvents(conn);

        List<Map<String, String>> errorEvents =
                events.stream().filter(e -> "error".equals(e.get("event"))).toList();
        assertEquals(1, errorEvents.size(), "Malformed parent should produce a single error event");
        assertTrue(
                errorEvents.get(0).get("data").contains("malformed braintrust parent"),
                "Error message should mention the malformed parent");

        // A malformed parent should abort before any experiment is created or summarized.
        assertTrue(
                events.stream().noneMatch(e -> "summary".equals(e.get("event"))),
                "Malformed parent should not produce a summary event");
    }

    @Test
    void testEvaluatorNotFound() throws Exception {
        EvalRequest request = new EvalRequest();
        request.setName("non-existent-eval");

        EvalRequest.DataSpec dataSpec = new EvalRequest.DataSpec();
        EvalRequest.EvalCaseData case1 = new EvalRequest.EvalCaseData();
        case1.setInput("test");
        dataSpec.setData(List.of(case1));
        request.setData(dataSpec);

        String requestJson = JSON_MAPPER.writeValueAsString(request);

        HttpClient client = HttpClient.newHttpClient();
        HttpRequest httpRequest =
                HttpRequest.newBuilder()
                        .uri(URI.create(TEST_URL + "/eval"))
                        .POST(HttpRequest.BodyPublishers.ofString(requestJson))
                        .header("Content-Type", "application/json")
                        .header("x-bt-auth-token", testHarness.braintrustApiKey())
                        .header("x-bt-project-id", TestHarness.defaultProjectId())
                        .header("x-bt-org-name", TestHarness.defaultOrgName())
                        .build();

        HttpResponse<String> response =
                client.send(httpRequest, HttpResponse.BodyHandlers.ofString());

        assertEquals(404, response.statusCode());
        assertTrue(response.body().contains("Evaluator not found"));
    }

    @Test
    void testEvalMethodNotAllowed() throws Exception {
        HttpClient client = HttpClient.newHttpClient();
        HttpRequest request =
                HttpRequest.newBuilder().uri(URI.create(TEST_URL + "/eval")).GET().build();

        HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

        assertEquals(405, response.statusCode());
    }

    @Test
    void testListEndpoint() throws Exception {
        HttpClient client = HttpClient.newHttpClient();
        HttpRequest request =
                HttpRequest.newBuilder()
                        .uri(URI.create(TEST_URL + "/list"))
                        .GET()
                        .header("x-bt-auth-token", testHarness.braintrustApiKey())
                        .header("x-bt-project-id", TestHarness.defaultProjectId())
                        .header("x-bt-org-name", TestHarness.defaultOrgName())
                        .build();

        HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

        assertEquals(200, response.statusCode());
        assertEquals(
                "application/json", response.headers().firstValue("Content-Type").orElse(null));

        // Parse and validate JSON response
        JsonNode root = JSON_MAPPER.readTree(response.body());

        // Should have one evaluator
        assertTrue(root.has(REMOTE_EVAL_NAME));

        JsonNode eval = root.get(REMOTE_EVAL_NAME);

        assertFalse(
                eval.has("parameters"),
                "parameters field must be omitted (not null) when the eval has no parameters");

        // Check scores
        assertTrue(eval.has("scores"));
        JsonNode scores = eval.get("scores");
        assertEquals(1, scores.size());

        assertEquals("simple_scorer", scores.get(0).get("name").asText());
    }

    @Test
    void testListMethodNotAllowed() throws Exception {
        HttpClient client = HttpClient.newHttpClient();
        HttpRequest request =
                HttpRequest.newBuilder()
                        .uri(URI.create(TEST_URL + "/list"))
                        .POST(HttpRequest.BodyPublishers.noBody())
                        .build();

        HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

        assertEquals(405, response.statusCode());
    }

    @Test
    void testListEndpointWithCors() throws Exception {
        HttpClient client = HttpClient.newHttpClient();
        HttpRequest request =
                HttpRequest.newBuilder()
                        .uri(URI.create(TEST_URL + "/list"))
                        .GET()
                        .header("Origin", "https://www.braintrust.dev")
                        .header("x-bt-auth-token", testHarness.braintrustApiKey())
                        .header("x-bt-project-id", TestHarness.defaultProjectId())
                        .header("x-bt-org-name", TestHarness.defaultOrgName())
                        .build();

        HttpResponse<String> response = client.send(request, HttpResponse.BodyHandlers.ofString());

        assertEquals(200, response.statusCode());
        assertEquals(
                "https://www.braintrust.dev",
                response.headers().firstValue("Access-Control-Allow-Origin").orElse(null));
    }

    @Test
    void testTaskErrorHandling() throws Exception {
        // Send an eval request with two cases: one good input and one that triggers a task error
        EvalRequest evalRequest = new EvalRequest();
        evalRequest.setName(TASK_ERROR_EVAL_NAME);
        evalRequest.setStream(true);

        EvalRequest.DataSpec dataSpec = new EvalRequest.DataSpec();

        EvalRequest.EvalCaseData goodCase = new EvalRequest.EvalCaseData();
        goodCase.setInput("good-input");
        goodCase.setExpected("expected");

        EvalRequest.EvalCaseData badCase = new EvalRequest.EvalCaseData();
        badCase.setInput("bad-input");
        badCase.setExpected("expected");

        dataSpec.setData(List.of(goodCase, badCase));
        evalRequest.setData(dataSpec);

        Map<String, Object> parentSpec =
                Map.of(
                        "object_type", PLAYGROUND_PARENT.type(),
                        "object_id", PLAYGROUND_PARENT.id(),
                        "propagated_event",
                                Map.of("span_attributes", Map.of("generation", "test-gen-err")));
        evalRequest.setParent(parentSpec);

        String requestBody = JSON_MAPPER.writeValueAsString(evalRequest);

        HttpURLConnection conn =
                (HttpURLConnection) new URI(TEST_URL + "/eval").toURL().openConnection();
        conn.setRequestMethod("POST");
        conn.setRequestProperty("Content-Type", "application/json");
        conn.setRequestProperty("x-bt-auth-token", testHarness.braintrustApiKey());
        conn.setRequestProperty("x-bt-project-id", TestHarness.defaultProjectId());
        conn.setRequestProperty("x-bt-org-name", TestHarness.defaultOrgName());
        conn.setDoOutput(true);

        conn.getOutputStream().write(requestBody.getBytes(StandardCharsets.UTF_8));
        conn.getOutputStream().flush();

        // Read SSE response
        assertEquals(200, conn.getResponseCode());

        List<Map<String, String>> events = readSSEEvents(conn);

        // Should complete with summary + done (eval continues despite the task error)
        List<Map<String, String>> summaryEvents =
                events.stream().filter(e -> "summary".equals(e.get("event"))).toList();
        List<Map<String, String>> doneEvents =
                events.stream().filter(e -> "done".equals(e.get("event"))).toList();
        assertEquals(1, summaryEvents.size(), "Should have 1 summary event");
        assertEquals(1, doneEvents.size(), "Should have 1 done event");

        // Both cases should produce a progress event (error case sends progress with null output
        // so the Playground can link to the trace)
        List<Map<String, String>> progressEvents =
                events.stream().filter(e -> "progress".equals(e.get("event"))).toList();
        assertEquals(2, progressEvents.size(), "Both cases should send a progress event");

        // Verify summary includes the fallback score from scoreForTaskException (default 0.0)
        JsonNode summaryData = JSON_MAPPER.readTree(summaryEvents.get(0).get("data"));
        assertTrue(summaryData.has("scores"));
        JsonNode scores = summaryData.get("scores");
        assertTrue(scores.has("exact_match"), "Summary should include exact_match scorer");

        // Verify spans
        List<SpanData> allSpans = testHarness.awaitExportedSpans();
        // Filter eval spans belonging to this test: either has generation tag (success path)
        // or has error status with our specific message (error path, since setEvalSpanAttributes
        // is not reached when the task throws)
        List<SpanData> evalSpans =
                allSpans.stream()
                        .filter(s -> s.getName().equals("eval"))
                        .filter(
                                s -> {
                                    var attrs =
                                            s.getAttributes()
                                                    .get(
                                                            AttributeKey.stringKey(
                                                                    "braintrust.span_attributes"));
                                    boolean hasGenTag =
                                            attrs != null && attrs.contains("test-gen-err");
                                    boolean isOurError =
                                            s.getStatus().getStatusCode() == StatusCode.ERROR
                                                    && s.getStatus()
                                                            .getDescription()
                                                            .contains("task failed on bad-input");
                                    return hasGenTag || isOurError;
                                })
                        .toList();
        assertEquals(2, evalSpans.size(), "Should have 2 eval spans (one per case)");

        // Find the errored eval span
        var erroredEvalSpan =
                evalSpans.stream()
                        .filter(s -> s.getStatus().getStatusCode() == StatusCode.ERROR)
                        .findFirst()
                        .orElseThrow(() -> new AssertionError("expected an errored eval span"));
        assertTrue(
                erroredEvalSpan.getStatus().getDescription().contains("task failed on bad-input"),
                "eval span error should contain the exception message");

        // The errored eval span should have output: null
        @SuppressWarnings("unchecked")
        Map<String, Object> erroredOutputJson =
                fromJson(
                        erroredEvalSpan
                                .getAttributes()
                                .get(AttributeKey.stringKey("braintrust.output_json")),
                        Map.class);
        assertNull(erroredOutputJson.get("output"), "errored case output should be null");

        // Find the task span for the errored case (should have ERROR status)
        var erroredTaskSpan =
                allSpans.stream()
                        .filter(s -> s.getName().equals("task"))
                        .filter(s -> s.getStatus().getStatusCode() == StatusCode.ERROR)
                        .filter(
                                s ->
                                        s.getParentSpanContext()
                                                .getSpanId()
                                                .equals(
                                                        erroredEvalSpan
                                                                .getSpanContext()
                                                                .getSpanId()))
                        .findFirst()
                        .orElseThrow(() -> new AssertionError("expected an errored task span"));
        assertFalse(
                erroredTaskSpan.getEvents().isEmpty(), "task span should have exception events");
        assertTrue(
                erroredTaskSpan.getEvents().stream().anyMatch(e -> e.getName().equals("exception")),
                "task span should have an exception event");

        // The errored case should still have a score span (from scoreForTaskException default 0.0)
        // The score span is a child of the task span (since the task scope is still active when
        // runScoreForTaskException is called from the catch block)
        var erroredScoreSpans =
                allSpans.stream()
                        .filter(s -> s.getName().equals("score"))
                        .filter(
                                s ->
                                        s.getParentSpanContext()
                                                .getSpanId()
                                                .equals(
                                                        erroredTaskSpan
                                                                .getSpanContext()
                                                                .getSpanId()))
                        .toList();
        assertEquals(1, erroredScoreSpans.size(), "errored case should have a score span");
        @SuppressWarnings("unchecked")
        Map<String, Object> fallbackScoresJson =
                fromJson(
                        erroredScoreSpans
                                .get(0)
                                .getAttributes()
                                .get(AttributeKey.stringKey("braintrust.scores")),
                        Map.class);
        assertEquals(
                0.0,
                ((Number) fallbackScoresJson.get("exact_match")).doubleValue(),
                "scoreForTaskException default should produce 0.0");

        // Verify the successful case has a non-error eval span
        var successEvalSpan =
                evalSpans.stream()
                        .filter(s -> s.getStatus().getStatusCode() != StatusCode.ERROR)
                        .findFirst()
                        .orElseThrow(() -> new AssertionError("expected a successful eval span"));
        @SuppressWarnings("unchecked")
        Map<String, Object> successOutputJson =
                fromJson(
                        successEvalSpan
                                .getAttributes()
                                .get(AttributeKey.stringKey("braintrust.output_json")),
                        Map.class);
        assertEquals(
                "result", successOutputJson.get("output"), "successful case should have output");
    }

    @Test
    void testScorerErrorHandling() throws Exception {
        // Send an eval request where one scorer throws
        EvalRequest evalRequest = new EvalRequest();
        evalRequest.setName(SCORER_ERROR_EVAL_NAME);
        evalRequest.setStream(true);

        EvalRequest.DataSpec dataSpec = new EvalRequest.DataSpec();
        EvalRequest.EvalCaseData case1 = new EvalRequest.EvalCaseData();
        case1.setInput("apple");
        case1.setExpected("fruit");
        dataSpec.setData(List.of(case1));
        evalRequest.setData(dataSpec);

        Map<String, Object> parentSpec =
                Map.of(
                        "object_type", PLAYGROUND_PARENT.type(),
                        "object_id", PLAYGROUND_PARENT.id(),
                        "propagated_event",
                                Map.of(
                                        "span_attributes",
                                        Map.of("generation", "test-gen-scorer-err")));
        evalRequest.setParent(parentSpec);

        String requestBody = JSON_MAPPER.writeValueAsString(evalRequest);

        HttpURLConnection conn =
                (HttpURLConnection) new URI(TEST_URL + "/eval").toURL().openConnection();
        conn.setRequestMethod("POST");
        conn.setRequestProperty("Content-Type", "application/json");
        conn.setRequestProperty("x-bt-auth-token", testHarness.braintrustApiKey());
        conn.setRequestProperty("x-bt-project-id", TestHarness.defaultProjectId());
        conn.setRequestProperty("x-bt-org-name", TestHarness.defaultOrgName());
        conn.setDoOutput(true);

        conn.getOutputStream().write(requestBody.getBytes(StandardCharsets.UTF_8));
        conn.getOutputStream().flush();

        // Read SSE response — eval should complete despite the broken scorer
        assertEquals(200, conn.getResponseCode());

        List<Map<String, String>> events = readSSEEvents(conn);

        // Should complete with summary + done
        List<Map<String, String>> summaryEvents =
                events.stream().filter(e -> "summary".equals(e.get("event"))).toList();
        List<Map<String, String>> doneEvents =
                events.stream().filter(e -> "done".equals(e.get("event"))).toList();
        assertEquals(1, summaryEvents.size(), "Should have 1 summary event");
        assertEquals(1, doneEvents.size(), "Should have 1 done event");

        // Verify summary has both scorers' results
        JsonNode summaryData = JSON_MAPPER.readTree(summaryEvents.get(0).get("data"));
        JsonNode summaryScores = summaryData.get("scores");

        // broken_scorer should have a fallback score of 0.0 (from scoreForScorerException default)
        assertTrue(summaryScores.has("broken_scorer"), "Summary should have broken_scorer");
        assertEquals(
                0.0,
                summaryScores.get("broken_scorer").get("score").asDouble(),
                0.001,
                "broken_scorer should fall back to 0.0");

        // working_scorer should have score 1.0
        assertTrue(summaryScores.has("working_scorer"), "Summary should have working_scorer");
        assertEquals(
                1.0,
                summaryScores.get("working_scorer").get("score").asDouble(),
                0.001,
                "working_scorer should produce 1.0");

        // Verify spans
        List<SpanData> allSpans = testHarness.awaitExportedSpans();

        // Filter score spans for this eval by generation tag on the parent eval span
        var evalSpans =
                allSpans.stream()
                        .filter(s -> s.getName().equals("eval"))
                        .filter(
                                s -> {
                                    var attrs =
                                            s.getAttributes()
                                                    .get(
                                                            AttributeKey.stringKey(
                                                                    "braintrust.span_attributes"));
                                    return attrs != null && attrs.contains("test-gen-scorer-err");
                                })
                        .toList();
        assertEquals(1, evalSpans.size(), "Should have 1 eval span");
        var evalSpan = evalSpans.get(0);

        // Find score spans under this eval
        var scoreSpans =
                allSpans.stream()
                        .filter(s -> s.getName().equals("score"))
                        .filter(
                                s ->
                                        s.getParentSpanContext()
                                                .getSpanId()
                                                .equals(evalSpan.getSpanContext().getSpanId()))
                        .toList();
        assertEquals(2, scoreSpans.size(), "Should have 2 score spans (one per scorer)");

        // Find the broken scorer's span — should have ERROR status
        var brokenScoreSpan =
                scoreSpans.stream()
                        .filter(s -> s.getStatus().getStatusCode() == StatusCode.ERROR)
                        .findFirst()
                        .orElseThrow(() -> new AssertionError("expected an errored score span"));
        assertTrue(
                brokenScoreSpan.getStatus().getDescription().contains("scorer is broken"),
                "score span error should contain the exception message");
        assertTrue(
                brokenScoreSpan.getEvents().stream().anyMatch(e -> e.getName().equals("exception")),
                "score span should have an exception event");

        // The broken scorer should still have fallback scores (default 0.0)
        @SuppressWarnings("unchecked")
        Map<String, Object> brokenScoresJson =
                fromJson(
                        brokenScoreSpan
                                .getAttributes()
                                .get(AttributeKey.stringKey("braintrust.scores")),
                        Map.class);
        assertEquals(
                0.0,
                ((Number) brokenScoresJson.get("broken_scorer")).doubleValue(),
                "scoreForScorerException default should produce 0.0");

        // Find the working scorer's span — should NOT have ERROR status
        var workingScoreSpan =
                scoreSpans.stream()
                        .filter(s -> s.getStatus().getStatusCode() != StatusCode.ERROR)
                        .findFirst()
                        .orElseThrow(() -> new AssertionError("expected a working score span"));
        @SuppressWarnings("unchecked")
        Map<String, Object> workingScoresJson =
                fromJson(
                        workingScoreSpan
                                .getAttributes()
                                .get(AttributeKey.stringKey("braintrust.scores")),
                        Map.class);
        assertEquals(
                1.0,
                ((Number) workingScoresJson.get("working_scorer")).doubleValue(),
                "working scorer should produce 1.0");
    }

    @Test
    void testParameterDefaultsAndOverrides() throws Exception {
        EvalRequest evalRequest = new EvalRequest();
        evalRequest.setName(PARAM_EVAL_NAME);
        evalRequest.setStream(true);

        // Override temperature but let model use its default
        evalRequest.setParameters(Map.of("temperature", 0.9));

        EvalRequest.DataSpec dataSpec = new EvalRequest.DataSpec();
        EvalRequest.EvalCaseData case1 = new EvalRequest.EvalCaseData();
        case1.setInput("hello");
        case1.setExpected("world");
        dataSpec.setData(List.of(case1));
        evalRequest.setData(dataSpec);

        Map<String, Object> parentSpec =
                Map.of(
                        "object_type", PLAYGROUND_PARENT.type(),
                        "object_id", PLAYGROUND_PARENT.id(),
                        "propagated_event",
                                Map.of("span_attributes", Map.of("generation", "test-gen-params")));
        evalRequest.setParent(parentSpec);

        String requestBody = JSON_MAPPER.writeValueAsString(evalRequest);

        HttpURLConnection conn =
                (HttpURLConnection) new URI(TEST_URL + "/eval").toURL().openConnection();
        conn.setRequestMethod("POST");
        conn.setRequestProperty("Content-Type", "application/json");
        conn.setRequestProperty("x-bt-auth-token", testHarness.braintrustApiKey());
        conn.setRequestProperty("x-bt-project-id", TestHarness.defaultProjectId());
        conn.setRequestProperty("x-bt-org-name", TestHarness.defaultOrgName());
        conn.setDoOutput(true);

        conn.getOutputStream().write(requestBody.getBytes(StandardCharsets.UTF_8));
        conn.getOutputStream().flush();

        assertEquals(200, conn.getResponseCode());

        List<Map<String, String>> events = readSSEEvents(conn);

        List<Map<String, String>> progressEvents =
                events.stream().filter(e -> "progress".equals(e.get("event"))).toList();
        assertEquals(1, progressEvents.size(), "Should have 1 progress event");

        // Task echoes "model:temperature" — model should be default "gpt-4",
        // temperature should be the overridden 0.9
        JsonNode progressData = JSON_MAPPER.readTree(progressEvents.get(0).get("data"));
        String taskOutput = progressData.get("data").asText();
        assertEquals(
                "\"gpt-4:0.9\"",
                taskOutput,
                "Task should receive default model and overridden temperature");

        assertEquals(1, events.stream().filter(e -> "summary".equals(e.get("event"))).count());
        assertEquals(1, events.stream().filter(e -> "done".equals(e.get("event"))).count());
    }

    /** Helper to read SSE events from an HttpURLConnection response. */
    private List<Map<String, String>> readSSEEvents(HttpURLConnection conn) throws Exception {
        BufferedReader reader =
                new BufferedReader(
                        new InputStreamReader(conn.getInputStream(), StandardCharsets.UTF_8));

        List<Map<String, String>> events = new ArrayList<>();
        String line;
        String currentEvent = null;
        StringBuilder currentData = new StringBuilder();

        while ((line = reader.readLine()) != null) {
            if (line.startsWith("event: ")) {
                currentEvent = line.substring(7);
            } else if (line.startsWith("data: ")) {
                currentData.append(line.substring(6));
            } else if (line.isEmpty() && currentEvent != null) {
                events.add(Map.of("event", currentEvent, "data", currentData.toString()));
                currentEvent = null;
                currentData = new StringBuilder();
            }
        }
        reader.close();
        return events;
    }

    /** Resolve a function's ID by project name + slug. */
    private static String lookupFunctionId(String projectName, String slug) {
        var functionsApi =
                new dev.braintrust.openapi.api.FunctionsApi(
                        testHarness.braintrust().openApiClient());
        var response =
                functionsApi.getFunction(
                        1, null, null, null, null, projectName, null, slug, null, null, null);
        var objects = response.getObjects();
        if (objects == null || objects.isEmpty()) {
            throw new IllegalStateException("function not found for slug: " + slug);
        }
        return objects.get(0).getId().toString();
    }

    // Code scorer source matching ScorerBrainstoreImplTest so the recorded cassettes are shared.
    // First element is the oldest version (always returns 0.0); the last (latest) returns 1.0 for
    // an exact match and 0.0 otherwise.
    private static final List<String> REMOTE_SCORER_CODE =
            List.of(
                    // language=typescript
                    """
import type { Trace } from 'braintrust';
// an older buggy version that always returns 0.0
async function handler({
  input,
  output,
  expected,
  metadata,
  trace,
}: {
  input: any;
  output: any;
  expected: any;
  metadata: Record<string, any>;
  trace: Trace;
}): Promise<
  | number
  | { score: number; name?: string; metadata?: Record<string, unknown> }
  | null
> {
  if (expected === null) return null;

  return {
    name: "typescript exact match",
    score: 0.0
  };
}
""",
                    // language=typescript
                    """
import type { Trace } from 'braintrust';
// returns 1.0 for exact match, 0.0 otherwise
async function handler({
  input,
  output,
  expected,
  metadata,
  trace,
}: {
  input: any;
  output: any;
  expected: any;
  metadata: Record<string, any>;
  trace: Trace;
}): Promise<
  | number
  | { score: number; name?: string; metadata?: Record<string, unknown> }
  | null
> {
  if (expected === null) return null;

  return {
    name: "typescript exact match",
    score: output === expected ? 1.0 : 0.0
  };
}
""");
}