From a77c91074d1d7d30c7f0baa63a6cd6605fa2198f Mon Sep 17 00:00:00 2001 From: Justin Bowen Date: Tue, 18 Aug 2026 20:35:29 -0700 Subject: [PATCH] fix: record the user turn an agent renders from its template MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Traces from a template-rendering agent carried the system prompt and the completion but not the user message. That is the half an evaluation scores: what the model was actually asked. The gap was easy to miss precisely because the other two attributes were present, so a trace looked populated. The instrumentation serializes prompt_options[:messages] — the turns a caller passed explicitly. An agent written the idiomatic way, `instructions:` plus `locals:` with the user turn in the action's ERB, has none at that point: the rendering happens later, in prepare_prompt_parameters. So the message reached the model and never reached the trace. Falls back to the rendered parameters when no explicit messages exist. prepare_prompt_parameters is a pure function of prompt_options — it deep_dups its input and mutates no instance state — so reading it here is safe. It does re-render the templates, which is why the fallback is only reached when there is nothing else to record. It never raises: an agent whose templates need context this call cannot supply loses the attribute, not the generation. Also handles message objects that respond to #content, not just hashes and strings, since that is what the rendered parameters contain. Verified against a self-hosted dashboard: a document-enrichment agent whose user turn is a rendered page-text template now shows System / User / Assistant in the trace's Conversation tab, where the User row was previously absent. An app-side workaround for this in ApplicationAgent has been removed and the behaviour holds. Co-Authored-By: Claude Opus 5 --- lib/active_agent/telemetry/instrumentation.rb | 35 +++++++++- test/telemetry/rendered_messages_test.rb | 64 +++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 test/telemetry/rendered_messages_test.rb diff --git a/lib/active_agent/telemetry/instrumentation.rb b/lib/active_agent/telemetry/instrumentation.rb index 2b2934c2..a94faa12 100644 --- a/lib/active_agent/telemetry/instrumentation.rb +++ b/lib/active_agent/telemetry/instrumentation.rb @@ -94,12 +94,24 @@ def process_prompt prompt_span.set_attribute("prompt.input.tools", JSON.generate(roster)) if roster.any? end - if (outbound = prompt_options[:messages]).present? + # prompt_options[:messages] holds the turns a caller passed + # explicitly. An agent that renders its user turn from the + # action's template — the idiomatic form, `instructions:` plus + # `locals:` — has none at this point: the rendering happens + # later, in prepare_prompt_parameters. Falling back to it means + # the message the model actually received is on the trace either + # way, which is what an evaluation scores. + outbound = prompt_options[:messages] + outbound = rendered_prompt_messages if outbound.blank? + + if outbound.present? serialized = Array(outbound).map { |message| if message.is_a?(Hash) role = message[:role] || message["role"] || "user" content = message[:content] || message["content"] { role: role.to_s, content: telemetry_truncate(content) } + elsif message.respond_to?(:content) + { role: (message.try(:role) || "user").to_s, content: telemetry_truncate(message.content) } else { role: "user", content: telemetry_truncate(message) } end @@ -257,6 +269,27 @@ def process_embed # tool loop) can't bloat the trace payload. TELEMETRY_ATTRIBUTE_MAX_CHARS = 4_000 + # The turns this generation will actually send, for an agent that + # renders its user message from the action's template rather than + # passing `messages:`. prepare_prompt_parameters is a pure function of + # prompt_options — it deep_dups its input and mutates no instance + # state — so calling it here is a read, not a side effect. It does + # re-render the templates, which is why it is only reached when there + # are no explicit messages to record. + # + # Never raises: a provider that builds parameters differently, or an + # agent whose templates need context this call does not have, must + # cost the generation nothing more than an absent attribute. + def rendered_prompt_messages + return unless respond_to?(:prepare_prompt_parameters, true) + + parameters = prepare_prompt_parameters + parameters[:messages] || parameters["messages"] + rescue StandardError => e + logger&.debug { "[ActiveAgent::Telemetry] could not read rendered messages: #{e.class}: #{e.message}" } + nil + end + def telemetry_truncate(value) text = value.to_s return text if text.length <= TELEMETRY_ATTRIBUTE_MAX_CHARS diff --git a/test/telemetry/rendered_messages_test.rb b/test/telemetry/rendered_messages_test.rb new file mode 100644 index 00000000..011da93a --- /dev/null +++ b/test/telemetry/rendered_messages_test.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +require "test_helper" + +# An agent that renders its user turn from the action's template — the +# idiomatic `instructions:` + `locals:` form — passes no `messages:`, so the +# instrumentation had nothing to record and `prompt.input.messages` was absent +# from its traces. The system prompt and the completion were both captured, +# which made the gap easy to miss: a trace looked populated while the half an +# evaluation scores, what the user actually said, was missing. +class RenderedMessagesTest < ActiveSupport::TestCase + class Recorder + attr_reader :attributes + + def initialize = @attributes = {} + def set_attribute(key, value) = @attributes[key] = value + end + + # Stands in for an agent whose messages only exist once the templates run. + class TemplateRenderingAgent + include ActiveAgent::Telemetry::Instrumentation::GenerationInstrumentation + + def initialize(rendered:, explicit: nil, raises: false) + @rendered = rendered + @explicit = explicit + @raises = raises + end + + def prompt_options = { messages: @explicit }.compact + + def prepare_prompt_parameters + raise "templates unavailable" if @raises + + { messages: @rendered } + end + + def logger = nil + end + + def messages_for(agent) + agent.send(:rendered_prompt_messages) + end + + test "reads the messages the templates rendered" do + agent = TemplateRenderingAgent.new(rendered: [ { role: "user", content: "rendered turn" } ]) + + assert_equal [ { role: "user", content: "rendered turn" } ], messages_for(agent) + end + + test "a failure to render costs the generation nothing but the attribute" do + agent = TemplateRenderingAgent.new(rendered: nil, raises: true) + + assert_nil messages_for(agent) + end + + test "an agent without prepare_prompt_parameters is left alone" do + bare = Class.new do + include ActiveAgent::Telemetry::Instrumentation::GenerationInstrumentation + def logger = nil + end.new + + assert_nil bare.send(:rendered_prompt_messages) + end +end