diff --git a/lib/active_agent/telemetry/instrumentation.rb b/lib/active_agent/telemetry/instrumentation.rb index 2b2934c2..a94faa12 100644 --- a/lib/active_agent/telemetry/instrumentation.rb +++ b/lib/active_agent/telemetry/instrumentation.rb @@ -94,12 +94,24 @@ def process_prompt prompt_span.set_attribute("prompt.input.tools", JSON.generate(roster)) if roster.any? end - if (outbound = prompt_options[:messages]).present? + # prompt_options[:messages] holds the turns a caller passed + # explicitly. An agent that renders its user turn from the + # action's template — the idiomatic form, `instructions:` plus + # `locals:` — has none at this point: the rendering happens + # later, in prepare_prompt_parameters. Falling back to it means + # the message the model actually received is on the trace either + # way, which is what an evaluation scores. + outbound = prompt_options[:messages] + outbound = rendered_prompt_messages if outbound.blank? + + if outbound.present? serialized = Array(outbound).map { |message| if message.is_a?(Hash) role = message[:role] || message["role"] || "user" content = message[:content] || message["content"] { role: role.to_s, content: telemetry_truncate(content) } + elsif message.respond_to?(:content) + { role: (message.try(:role) || "user").to_s, content: telemetry_truncate(message.content) } else { role: "user", content: telemetry_truncate(message) } end @@ -257,6 +269,27 @@ def process_embed # tool loop) can't bloat the trace payload. TELEMETRY_ATTRIBUTE_MAX_CHARS = 4_000 + # The turns this generation will actually send, for an agent that + # renders its user message from the action's template rather than + # passing `messages:`. prepare_prompt_parameters is a pure function of + # prompt_options — it deep_dups its input and mutates no instance + # state — so calling it here is a read, not a side effect. It does + # re-render the templates, which is why it is only reached when there + # are no explicit messages to record. + # + # Never raises: a provider that builds parameters differently, or an + # agent whose templates need context this call does not have, must + # cost the generation nothing more than an absent attribute. + def rendered_prompt_messages + return unless respond_to?(:prepare_prompt_parameters, true) + + parameters = prepare_prompt_parameters + parameters[:messages] || parameters["messages"] + rescue StandardError => e + logger&.debug { "[ActiveAgent::Telemetry] could not read rendered messages: #{e.class}: #{e.message}" } + nil + end + def telemetry_truncate(value) text = value.to_s return text if text.length <= TELEMETRY_ATTRIBUTE_MAX_CHARS diff --git a/test/telemetry/rendered_messages_test.rb b/test/telemetry/rendered_messages_test.rb new file mode 100644 index 00000000..011da93a --- /dev/null +++ b/test/telemetry/rendered_messages_test.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require "test_helper" + +# An agent that renders its user turn from the action's template — the +# idiomatic `instructions:` + `locals:` form — passes no `messages:`, so the +# instrumentation had nothing to record and `prompt.input.messages` was absent +# from its traces. The system prompt and the completion were both captured, +# which made the gap easy to miss: a trace looked populated while the half an +# evaluation scores, what the user actually said, was missing. +class RenderedMessagesTest < ActiveSupport::TestCase + class Recorder + attr_reader :attributes + + def initialize = @attributes = {} + def set_attribute(key, value) = @attributes[key] = value + end + + # Stands in for an agent whose messages only exist once the templates run. + class TemplateRenderingAgent + include ActiveAgent::Telemetry::Instrumentation::GenerationInstrumentation + + def initialize(rendered:, explicit: nil, raises: false) + @rendered = rendered + @explicit = explicit + @raises = raises + end + + def prompt_options = { messages: @explicit }.compact + + def prepare_prompt_parameters + raise "templates unavailable" if @raises + + { messages: @rendered } + end + + def logger = nil + end + + def messages_for(agent) + agent.send(:rendered_prompt_messages) + end + + test "reads the messages the templates rendered" do + agent = TemplateRenderingAgent.new(rendered: [ { role: "user", content: "rendered turn" } ]) + + assert_equal [ { role: "user", content: "rendered turn" } ], messages_for(agent) + end + + test "a failure to render costs the generation nothing but the attribute" do + agent = TemplateRenderingAgent.new(rendered: nil, raises: true) + + assert_nil messages_for(agent) + end + + test "an agent without prepare_prompt_parameters is left alone" do + bare = Class.new do + include ActiveAgent::Telemetry::Instrumentation::GenerationInstrumentation + def logger = nil + end.new + + assert_nil bare.send(:rendered_prompt_messages) + end +end