Skip to content

feat: add debug traces and failure events to message history #238

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
pirate-bot merged 1 commit into from
Aug 31, 2026
Merged

feat: add debug traces and failure events to message history #238

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
505 changes: 503 additions & 2 deletions inc/API.php
View file
Open in desktop

Large diffs are not rendered by default.

23 changes: 21 additions & 2 deletions inc/OpenAI.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -301,6 +301,17 @@ public static function resolve_chat_model( $model ) {
return $model;
}

/**
* Get the effective chat model used for completions.
*
* @since 2.1.0
*
* @return string
*/
public function get_chat_model() {
return $this->chat_model;
}

/**
* Set whether request errors should be persisted as a service error notice.
*
Expand Down Expand Up @@ -823,7 +834,7 @@ public function get_response( $response_id ) {
* @param string $conversation Conversation id.
* @param callable $on_delta Receives each text delta (string).
*
* @return array{id:string,text:string,tool_calls:array<int,array{call_id:string,name:string,arguments:string}>}|\WP_Error
* @return array{id:string,text:string,tool_calls:array<int,array{call_id:string,name:string,arguments:string}>,usage:object|null}|\WP_Error
*/
public function stream_response( $items, $conversation, $on_delta ) {
if ( ! $this->api_key ) {
Expand All @@ -848,8 +859,9 @@ public function stream_response( $items, $conversation, $on_delta ) {
$sse_buffer = '';
$stream_err = null;
$tool_calls = [];
$usage = null;

$write = function ( $ch, $chunk ) use ( &$sse_buffer, &$assembled, &$response_id, &$stream_err, &$tool_calls, $on_delta ) {
$write = function ( $ch, $chunk ) use ( &$sse_buffer, &$assembled, &$response_id, &$stream_err, &$tool_calls, &$usage, $on_delta ) {
$sse_buffer .= $chunk;

while ( false !== ( $pos = strpos( $sse_buffer, "\n\n" ) ) ) {
Expand Down Expand Up @@ -886,6 +898,12 @@ public function stream_response( $items, $conversation, $on_delta ) {
$response_id = $event->response->id;
}

// The terminal response.completed event carries the run's token
// usage; kept for the per-message debug trace.
if ( isset( $event->response->usage ) ) {
$usage = $event->response->usage;
}

if ( 'response.output_text.delta' === $event->type && isset( $event->delta ) && is_string( $event->delta ) ) {
$assembled .= $event->delta;
call_user_func( $on_delta, $event->delta );
Expand Down Expand Up @@ -959,6 +977,7 @@ public function stream_response( $items, $conversation, $on_delta ) {
'id' => $response_id,
'text' => $assembled,
'tool_calls' => $tool_calls,
'usage' => $usage,
];
}

Expand Down
67 changes: 61 additions & 6 deletions inc/Stream.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,7 @@ public function stream() {
$is_test = ! empty( $job['is_test'] );
$source_post_ids = isset( $job['source_post_ids'] ) ? $job['source_post_ids'] : [];
$page = isset( $job['page'] ) && is_array( $job['page'] ) ? $job['page'] : null;
$debug = isset( $job['debug'] ) && is_array( $job['debug'] ) ? $job['debug'] : [];

Main::add_labels_to_default_settings();
$settings = Main::get_settings();
Expand All @@ -183,7 +184,7 @@ public function stream() {
$this->open_stream();

if ( Hyve_Connect::is_active() ) {
$this->stream_connect( $message, $thread_id, $record_id, $is_test, $settings, $default_message, $page );
$this->stream_connect( $message, $thread_id, $record_id, $is_test, $settings, $default_message, $page, $debug );
exit;
}

Expand Down Expand Up @@ -214,9 +215,13 @@ public function stream() {
}
};

$result = null;
$cancelled = false;
$used_tools = false;
$result = null;
$cancelled = false;
$used_tools = false;
$tool_rounds = 0;
$usage_in = 0;
$usage_out = 0;
$has_usage = false;

for ( $i = 0; ; $i++ ) {
$raw = '';
Expand All @@ -226,10 +231,24 @@ public function stream() {
$result = OpenAI::instance()->stream_response( $items, $thread_id, $on_delta );

if ( is_wp_error( $result ) ) {
// The user message is only recorded when a reply lands, so a
// failed stream must record the turn itself: the message plus
// an error event, never a bot bubble.
API::instance()->record_chat_failure( $thread_id, $record_id, $message, $is_test, (string) $result->get_error_code(), $result->get_error_message() );
$this->send_event( 'error', [ 'message' => $result->get_error_message() ] );
exit;
}

// Sum usage across tool round-trips so the recorded trace reflects
// the whole turn, not only the closing response.
$run_usage = API::normalize_usage( isset( $result['usage'] ) ? $result['usage'] : null );

if ( null !== $run_usage ) {
$has_usage = true;
$usage_in += $run_usage['input'];
$usage_out += $run_usage['output'];
}

$tool_calls = $result['tool_calls'];

if ( empty( $tool_calls ) || $cancelled ) {
Expand Down Expand Up @@ -263,7 +282,8 @@ public function stream() {
$this->send_event( 'status', [ 'state' => 'tool' ] );

$used_tools = true;
$items = $outputs;
++$tool_rounds;
$items = $outputs;
}

if ( $cancelled ) {
Expand Down Expand Up @@ -357,6 +377,29 @@ public function stream() {
$final = $data['message'];
}

$extra = [
'mode' => 'self_hosted',
'transport' => 'stream',
'tools_used' => $used_tools,
];

if ( 0 < $tool_rounds ) {
$extra['tool_iterations'] = $tool_rounds;
}

if ( $cancelled ) {
$extra['tools_aborted'] = true;
}

if ( $has_usage ) {
$extra['usage'] = [
'input' => $usage_in,
'output' => $usage_out,
];
}

$payload['debug'] = API::finalize_chat_debug( $debug, $extra, $payload );

if ( ! $is_test ) {
do_action( 'hyve_chat_response', $result['id'], $thread_id, $message, $record_id, $payload, $final );
}
Expand All @@ -381,10 +424,11 @@ public function stream() {
* @param array<string, mixed> $settings Plugin settings.
* @param string $default_message Fallback shown when the model cannot answer.
* @param array<string, mixed>|null $page Current page payload (see Page_Context::payload()).
* @param array<string, mixed> $debug Debug trace collected when the turn was prepared.
*
* @return void
*/
private function stream_connect( $message, $thread_id, $record_id, $is_test, $settings, $default_message, $page = null ) {
private function stream_connect( $message, $thread_id, $record_id, $is_test, $settings, $default_message, $page = null, $debug = [] ) {
$sources = [];

$on_event = function ( $event, $data ) use ( &$sources ) {
Expand Down Expand Up @@ -412,6 +456,8 @@ private function stream_connect( $message, $thread_id, $record_id, $is_test, $se
$result = Hyve_Connect::instance()->stream_chat( $payload, $on_event );

if ( is_wp_error( $result ) ) {
API::instance()->record_chat_failure( $thread_id, $record_id, $message, $is_test, (string) $result->get_error_code(), $result->get_error_message() );

$error_code = $result->get_error_code();

// An empty/purged KB is visitor-facing parity with self-hosted: show
Expand Down Expand Up @@ -488,6 +534,15 @@ private function stream_connect( $message, $thread_id, $record_id, $is_test, $se
$final = $data['message'];
}

$payload['debug'] = API::finalize_chat_debug(
array_merge( $debug, API::instance()->connect_debug( $result ) ),
[
'mode' => 'connect',
'transport' => 'stream',
],
$payload
);

if ( ! $is_test ) {
do_action( 'hyve_chat_response', $thread, $thread, $message, $record_id, $payload, $final );
}
Expand Down
27 changes: 25 additions & 2 deletions inc/Threads.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,13 +84,23 @@ public function record_message( $run_id, $thread_id, $query, $record_id, $messag
return;
}

// The debug trace explains the reply after the fact: whether the model
// answered, what the retrieval matched and which tools ran. Without it a
// refusal is indistinguishable from a genuine knowledge gap.
$debug = isset( $message['debug'] ) && is_array( $message['debug'] ) ? $message['debug'] : [];

if ( isset( $message['success'] ) ) {
$debug['answered'] = (bool) $message['success'];
}

self::add_message(
intval( $record_id ),
[
'thread_id' => $thread_id,
'sender' => 'bot',
'message' => wp_kses_post( $response ),
'display' => isset( $message['display'] ) && is_array( $message['display'] ) ? $message['display'] : null,
'debug' => ! empty( $debug ) ? $debug : null,
]
);
}
Expand Down Expand Up @@ -131,7 +141,9 @@ public function record_thread( $thread_id, $record_id, $message ) {

/**
* Build a stored transcript entry, carrying an optional `display` (skill
* cards or choices) alongside the message so history matches what was shown.
* cards or choices) alongside the message so history matches what was shown,
* and an optional `debug` trace (answered flag, retrieval sources and
* scores, tools) so the reply can be explained after the fact.
*
* @param array<string, mixed> $data The message data.
*
Expand All @@ -148,6 +160,10 @@ private static function build_entry( $data ) {
$entry['display'] = $data['display'];
}

if ( ! empty( $data['debug'] ) && is_array( $data['debug'] ) ) {
$entry['debug'] = $data['debug'];
}

return $entry;
}

Expand Down Expand Up @@ -205,7 +221,14 @@ public static function add_message( $post_id, $data ) {
$thread_id = get_post_meta( $post_id, '_hyve_thread_id', true );

if ( $thread_id !== $data['thread_id'] ) {
return self::create_thread( $data['message'], $data );
// A failed first turn records the thread before any conversation
// is minted, leaving an empty placeholder id. Adopt the real id
// on the next successful turn instead of forking a new thread.
if ( '' === (string) $thread_id && '' !== (string) $data['thread_id'] ) {
update_post_meta( $post_id, '_hyve_thread_id', $data['thread_id'] );
} else {
return self::create_thread( $data['message'], $data );
}
}

$thread_data = get_post_meta( $post_id, '_hyve_thread_data', true );
Expand Down
Loading
Loading