From 7e9cf4276206b4182becffe7fd5b2e4e6d9ffeb0 Mon Sep 17 00:00:00 2001 From: alectimison-maker Date: Sun, 26 Jul 2026 14:31:36 +0800 Subject: [PATCH 1/8] fix(captcha): report response token state --- src/chrome/src/agent/captcha-frame-runtime.js | 10 ++ .../src/agent/captcha-frame-runtime.js | 10 ++ test/run.js | 119 +++++++++++++++++- 3 files changed, 138 insertions(+), 1 deletion(-) diff --git a/src/chrome/src/agent/captcha-frame-runtime.js b/src/chrome/src/agent/captcha-frame-runtime.js index 64675cc70..63f0fa6ac 100644 --- a/src/chrome/src/agent/captcha-frame-runtime.js +++ b/src/chrome/src/agent/captcha-frame-runtime.js @@ -300,6 +300,7 @@ function candidateSummary(candidate) { recaptchaDataSValue: candidate?.recaptchaDataSValue || null, explicitWebsiteKey: candidate?.explicitWebsiteKey === true, callbackName: candidate?.callbackName || null, + responseTokenPresent: candidate?.responseTokenPresent === true, responseFieldId: candidate?.responseFieldId || null, responseFieldIndex: Number.isInteger(candidate?.responseFieldIndex) ? candidate.responseFieldIndex @@ -407,6 +408,8 @@ export function selectCaptchaCandidate(candidates, constraints = {}) { challengeFrame: previous.challengeFrame === true || candidate.challengeFrame === true, dialogAssociated: previous.dialogAssociated === true || candidate.dialogAssociated === true, responseField: previous.responseField === true || candidate.responseField === true, + responseTokenPresent: previous.responseTokenPresent === true + || candidate.responseTokenPresent === true, }; for (const field of taskParameterFields) { const previousValue = previous[field]; @@ -686,6 +689,7 @@ export function detectCaptchaCandidatesInPage(scope = null) { const { responseFieldDialogAssociated, alsoResponseFieldDialogAssociated, + alsoResponseTokenPresent, ...serializableCandidate } = candidate; candidates.push({ @@ -693,6 +697,8 @@ export function detectCaptchaCandidatesInPage(scope = null) { frameUrl, challengeFrame, responseField, + responseTokenPresent: candidate.responseTokenPresent === true + || alsoResponseTokenPresent === true, documentTimeOrigin, dialogAssociated: candidate.dialogAssociated === true || responseFieldDialogAssociated === true @@ -728,9 +734,12 @@ export function detectCaptchaCandidatesInPage(scope = null) { const responseFieldIndex = fields.indexOf(field); let responseFieldId = ''; try { responseFieldId = String(field.id || field.getAttribute?.('id') || ''); } catch (_) {} + let responseTokenPresent = false; + try { responseTokenPresent = String(field.value || '').trim().length > 0; } catch (_) {} return { ...(responseFieldId ? { responseFieldId } : {}), ...(responseFieldIndex >= 0 ? { responseFieldIndex } : {}), + responseTokenPresent, ...(elementInChallengeDialog(field) ? { responseFieldDialogAssociated: true } : {}), }; }; @@ -741,6 +750,7 @@ export function detectCaptchaCandidatesInPage(scope = null) { ...(Number.isInteger(identity.responseFieldIndex) ? { alsoResponseFieldIndex: identity.responseFieldIndex } : {}), + alsoResponseTokenPresent: identity.responseTokenPresent === true, ...(identity.responseFieldDialogAssociated ? { alsoResponseFieldDialogAssociated: true } : {}), diff --git a/src/firefox/src/agent/captcha-frame-runtime.js b/src/firefox/src/agent/captcha-frame-runtime.js index 64675cc70..63f0fa6ac 100644 --- a/src/firefox/src/agent/captcha-frame-runtime.js +++ b/src/firefox/src/agent/captcha-frame-runtime.js @@ -300,6 +300,7 @@ function candidateSummary(candidate) { recaptchaDataSValue: candidate?.recaptchaDataSValue || null, explicitWebsiteKey: candidate?.explicitWebsiteKey === true, callbackName: candidate?.callbackName || null, + responseTokenPresent: candidate?.responseTokenPresent === true, responseFieldId: candidate?.responseFieldId || null, responseFieldIndex: Number.isInteger(candidate?.responseFieldIndex) ? candidate.responseFieldIndex @@ -407,6 +408,8 @@ export function selectCaptchaCandidate(candidates, constraints = {}) { challengeFrame: previous.challengeFrame === true || candidate.challengeFrame === true, dialogAssociated: previous.dialogAssociated === true || candidate.dialogAssociated === true, responseField: previous.responseField === true || candidate.responseField === true, + responseTokenPresent: previous.responseTokenPresent === true + || candidate.responseTokenPresent === true, }; for (const field of taskParameterFields) { const previousValue = previous[field]; @@ -686,6 +689,7 @@ export function detectCaptchaCandidatesInPage(scope = null) { const { responseFieldDialogAssociated, alsoResponseFieldDialogAssociated, + alsoResponseTokenPresent, ...serializableCandidate } = candidate; candidates.push({ @@ -693,6 +697,8 @@ export function detectCaptchaCandidatesInPage(scope = null) { frameUrl, challengeFrame, responseField, + responseTokenPresent: candidate.responseTokenPresent === true + || alsoResponseTokenPresent === true, documentTimeOrigin, dialogAssociated: candidate.dialogAssociated === true || responseFieldDialogAssociated === true @@ -728,9 +734,12 @@ export function detectCaptchaCandidatesInPage(scope = null) { const responseFieldIndex = fields.indexOf(field); let responseFieldId = ''; try { responseFieldId = String(field.id || field.getAttribute?.('id') || ''); } catch (_) {} + let responseTokenPresent = false; + try { responseTokenPresent = String(field.value || '').trim().length > 0; } catch (_) {} return { ...(responseFieldId ? { responseFieldId } : {}), ...(responseFieldIndex >= 0 ? { responseFieldIndex } : {}), + responseTokenPresent, ...(elementInChallengeDialog(field) ? { responseFieldDialogAssociated: true } : {}), }; }; @@ -741,6 +750,7 @@ export function detectCaptchaCandidatesInPage(scope = null) { ...(Number.isInteger(identity.responseFieldIndex) ? { alsoResponseFieldIndex: identity.responseFieldIndex } : {}), + alsoResponseTokenPresent: identity.responseTokenPresent === true, ...(identity.responseFieldDialogAssociated ? { alsoResponseFieldDialogAssociated: true } : {}), diff --git a/test/run.js b/test/run.js index 8d284e3c0..be22bba29 100644 --- a/test/run.js +++ b/test/run.js @@ -52554,6 +52554,7 @@ function captchaEl(tag, attrs = {}, children = []) { src: attrs.src, innerText: attrs.innerText || '', textContent: attrs.textContent || attrs.innerText || '', + value: attrs.value || '', hidden: attrs.hidden === true, style: {}, classList: { contains: (c) => String(attrs.class || '').split(/\s+/).includes(c) }, @@ -52685,6 +52686,13 @@ async function detectCaptchaOnFakePage(build, nodes) { }); } +async function detectCaptchaDetailsOnFakePage(build, nodes) { + return withCaptchaFakePage(build, nodes, async () => { + const mod = await import(pathToFileURL(path.join(ROOT, `src/${build}/src/agent/captcha-solver.js`)).href); + return mod.detectCaptcha(1); + }); +} + test('challenge-dialog routing detects supported widgets and diagnoses unsupported Arkose frames', async () => { for (const [build, AgentClass] of [['chrome', AgentCh], ['firefox', AgentFx]]) { const supportedNodes = [ @@ -53136,12 +53144,17 @@ test('challenge dialog with no enabled supported solver stops the batch for manu } }); -test('CAPTCHA gate helper stays byte-identical and cloud traces retain gate diagnostics outside update rollover', () => { +test('CAPTCHA helpers stay byte-identical and cloud traces retain gate diagnostics outside update rollover', () => { assert.equal( fs.readFileSync(path.join(ROOT, 'src/chrome/src/agent/captcha-gate.js'), 'utf8'), fs.readFileSync(path.join(ROOT, 'src/firefox/src/agent/captcha-gate.js'), 'utf8'), 'chrome and firefox CAPTCHA gate helpers must remain byte-identical', ); + assert.equal( + fs.readFileSync(path.join(ROOT, 'src/chrome/src/agent/captcha-frame-runtime.js'), 'utf8'), + fs.readFileSync(path.join(ROOT, 'src/firefox/src/agent/captcha-frame-runtime.js'), 'utf8'), + 'chrome and firefox CAPTCHA frame runtimes must remain byte-identical', + ); const cloudSource = fs.readFileSync(path.join(ROOT, 'src/chrome/src/cloud-runs.js'), 'utf8'); assert.match(cloudSource, /if \(type === 'captcha_gate'\)[\s\S]*?run\.captchaDiagnostics = \{ \.\.\.scrubbedData, observedAt: run\.updatedAt \};/); assert.match(cloudSource, /\.\.\.\(run\.captchaDiagnostics \? \{ captchaDiagnostics: run\.captchaDiagnostics \} : \{\}\)/); @@ -53401,6 +53414,110 @@ test('captcha detection binds the selected widget to its own response field', as } }); +test('captcha detection reports exact response token state without exposing token values', async () => { + const token = 'RAW_CAPTCHA_TOKEN_MUST_NOT_ESCAPE_7fc2'; + for (const build of ['chrome', 'firefox']) { + const solvedField = captchaEl('textarea', { + id: 'g-recaptcha-response-solved', + name: 'g-recaptcha-response', + value: token, + }); + const pendingField = captchaEl('textarea', { + id: 'g-recaptcha-response-pending', + name: 'g-recaptcha-response', + }); + const solvedHost = captchaEl('div', { + class: 'g-recaptcha', + 'data-sitekey': 'KEY_SOLVED_WIDGET', + hidden: true, + }, [solvedField]); + const pendingHost = captchaEl('div', { + class: 'g-recaptcha', + 'data-sitekey': 'KEY_PENDING_WIDGET', + }, [pendingField]); + + const pendingDetection = await detectCaptchaDetailsOnFakePage(build, [ + solvedHost, + pendingHost, + captchaEl('script', { src: 'https://www.google.com/recaptcha/api.js' }), + ]); + assert.equal( + pendingDetection.selected.responseTokenPresent, + false, + `${build}: a solved sibling widget contaminated the selected pending widget`, + ); + assert.equal( + pendingDetection.candidates.find(candidate => + candidate.websiteKey === 'KEY_SOLVED_WIDGET' + )?.responseTokenPresent, + true, + `${build}: non-empty exact response field was not reported as solved`, + ); + assert.doesNotMatch( + JSON.stringify(pendingDetection), + new RegExp(token), + `${build}: raw response token escaped through detection`, + ); + + pendingField.value = token; + const solvedDetection = await detectCaptchaDetailsOnFakePage(build, [ + solvedHost, + pendingHost, + captchaEl('script', { src: 'https://www.google.com/recaptcha/api.js' }), + ]); + assert.equal( + solvedDetection.selected.responseTokenPresent, + true, + `${build}: selected non-empty response field was not reported as solved`, + ); + assert.doesNotMatch( + JSON.stringify(solvedDetection), + new RegExp(token), + `${build}: selected raw response token escaped through detection`, + ); + + const primaryField = captchaEl('textarea', { + id: 'h-captcha-response-primary', + name: 'h-captcha-response', + }); + const compatibilityField = captchaEl('textarea', { + id: 'g-recaptcha-response-compatibility', + name: 'g-recaptcha-response', + value: token, + }); + const hcaptcha = await detectCaptchaOnFakePage(build, [ + captchaEl('div', { + class: 'h-captcha', + 'data-sitekey': 'HKEY_SOLVED_COMPATIBILITY', + }, [primaryField, compatibilityField]), + ]); + assert.equal( + hcaptcha.responseTokenPresent, + true, + `${build}: solved hCaptcha compatibility field was ignored`, + ); + assert.doesNotMatch( + JSON.stringify(hcaptcha), + new RegExp(token), + `${build}: hCaptcha compatibility token escaped through detection`, + ); + + Object.defineProperty(pendingField, 'value', { + configurable: true, + get() { throw new Error('hostile response field getter'); }, + }); + const unreadable = await detectCaptchaOnFakePage(build, [ + pendingHost, + captchaEl('script', { src: 'https://www.google.com/recaptcha/api.js' }), + ]); + assert.equal( + unreadable.responseTokenPresent, + false, + `${build}: unreadable response field did not fail closed`, + ); + } +}); + test('captcha detection ranks a visible nested v2 Enterprise challenge above background v3', async () => { const topCandidate = { type: 'recaptcha_v3_enterprise', From f4fc92bc27ce50647ab1b2a3aa531d6ba3625515 Mon Sep 17 00:00:00 2001 From: alectimison-maker Date: Sun, 26 Jul 2026 14:51:59 +0800 Subject: [PATCH 2/8] refactor(agent): extract text tool-call parser --- src/chrome/src/agent/agent.js | 132 +--------------------- src/chrome/src/agent/tool-call-parser.js | 123 ++++++++++++++++++++ src/firefox/src/agent/agent.js | 120 +------------------- src/firefox/src/agent/tool-call-parser.js | 123 ++++++++++++++++++++ test/run.js | 103 +++++++++++++++++ 5 files changed, 353 insertions(+), 248 deletions(-) create mode 100644 src/chrome/src/agent/tool-call-parser.js create mode 100644 src/firefox/src/agent/tool-call-parser.js diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index d7c39a485..34f39248f 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -1,6 +1,7 @@ import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, getToolsForMode, SYSTEM_PROMPT_ASK, SYSTEM_PROMPT_ACT, SYSTEM_PROMPT_ACT_COMPACT, SYSTEM_PROMPT_ACT_MID, SYSTEM_PROMPT_DEV_APPENDIX, SYSTEM_PROMPT_WEBMCP_ASK, SYSTEM_PROMPT_WEBMCP_ACT } from './tools.js'; import { handleDoneJson } from './cloud-output.js'; import { LoopDetector } from './loop-detector.js'; +import { parseToolCallsFromText } from './tool-call-parser.js'; import { BROWSER_MUTATION_TOOLS, STATE_CHANGE_TOOLS as SHARED_STATE_CHANGE_TOOLS } from './mutation-tools.js'; import { isCredentialField, CREDENTIAL_NOTE_STRICT, STRICT_SECRET_SYSTEM_NOTE } from './credential-fields.js'; import { detectProgressAction, formatLedgerRow, formatLedgerSummary, isBlockedLedgerDowngrade, isTerminalLedgerStatus, isValidLedgerStatus, ledgerDoneBlock, ledgerRowKey, normalizeLedgerStatus, progressCounts, selectLedgerRows, unresolvedLedgerRows, upsertLedgerItems } from './progress-ledger.js'; @@ -18715,136 +18716,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d * smaller set (e.g. COMPACT_TOOL_NAMES) to restrict in compact mode. */ _tryParseToolCallsFromText(text, allowedNames = AGENT_TOOL_NAMES) { - if (!text || text.length > 10000) return []; - - const results = []; - const parseXmlParamValue = (value) => { - const cleaned = String(value || '') - .replace(/<[^>]+>/g, '') - .trim(); - if (!cleaned) return ''; - try { - if (/^(?:"|'.*'|\{|\[|-?\d|true\b|false\b|null\b)/i.test(cleaned)) { - return JSON.parse(cleaned.replace(/^'([\s\S]*)'$/, '"$1"')); - } - } catch { /* fall through to string cleanup */ } - return cleaned.replace(/^["']+|["']+$/g, ''); - }; - - // Collect candidate JSON strings from known wrapper patterns. - const patterns = [ - // JSON - /\s*([\s\S]*?)\s*<\/tool_call>/gi, - // <|tool_call|>JSON<|/tool_call|> or <|tool_call>JSON - /<\|tool_call\|?>\s*([\s\S]*?)\s*<\|?\/?tool_call\|?>/gi, - // JSON - /\s*([\s\S]*?)\s*<\/functioncall>/gi, - ]; - - for (const re of patterns) { - let m; - while ((m = re.exec(text)) !== null) { - const inner = m[1].trim(); - // Try JSON first (most common). - try { - const obj = JSON.parse(inner); - if (obj && obj.name && allowedNames.has(obj.name)) { - results.push(obj); - continue; - } - } catch { /* not JSON — try call:name{} format below */ } - - // call:toolName{key:<|"|>value<|"|>, ...} format. - // Some local models use <|"|> as quote tokens and call:name as the - // invocation syntax. Normalize to JSON and parse. - const callMatch = /^call:(\w+)\s*\{([\s\S]*)\}$/.exec(inner); - if (callMatch && allowedNames.has(callMatch[1])) { - const toolName = callMatch[1]; - let argsBody = callMatch[2] - .replace(/<\|"\|>/g, '"') // replace quote tokens with real quotes - .replace(/<\|'\\?\|>/g, "'"); // handle single-quote tokens if any - // argsBody is now like: url:"https://example.com",text:"hello" - // Wrap unquoted keys to make valid JSON: key:"val" → "key":"val" - argsBody = argsBody.replace(/(?<=^|,)\s*(\w+)\s*:/g, '"$1":'); - try { - const args = JSON.parse(`{${argsBody}}`); - results.push({ name: toolName, arguments: args }); - } catch { - // If JSON parse still fails, try treating entire body as single - // string argument for zero-arg or simple calls. - results.push({ name: toolName, arguments: {} }); - } - continue; - } - } - } - - // XML-ish tool-call format used by some local/chat-template models: - // ref_6... - const xmlToolRe = /\s*\s*([\s\S]*?)\s*<\/function>\s*<\/tool_call>/gi; - let xmlMatch; - while ((xmlMatch = xmlToolRe.exec(text)) !== null) { - const toolName = xmlMatch[1] || xmlMatch[2]; - if (!allowedNames.has(toolName)) continue; - const body = xmlMatch[3] || ''; - const args = {}; - const paramRe = /\s*([\s\S]*?)\s*<\/parameter>/gi; - let paramMatch; - while ((paramMatch = paramRe.exec(body)) !== null) { - const key = paramMatch[1] || paramMatch[2]; - if (!key) continue; - args[key] = parseXmlParamValue(paramMatch[3]); - } - results.push({ name: toolName, arguments: args }); - } - - // Fallback: scan for bare JSON objects containing a "name" key with a - // known tool name. Only look for top-level objects (starts with {). - if (results.length === 0) { - const bareRe = /\{[^{}]*"name"\s*:\s*"(\w+)"[^{}]*\}/g; - let m; - while ((m = bareRe.exec(text)) !== null) { - if (!allowedNames.has(m[1])) continue; - try { - const obj = JSON.parse(m[0]); - if (obj && obj.name && allowedNames.has(obj.name)) { - results.push(obj); - } - } catch { /* skip */ } - } - } - - // Last resort: call:toolName{...} outside of any wrapper tags. - if (results.length === 0) { - const callRe = /call:(\w+)\s*\{([\s\S]*?)\}/g; - let m; - while ((m = callRe.exec(text)) !== null) { - if (!allowedNames.has(m[1])) continue; - const toolName = m[1]; - let argsBody = m[2] - .replace(/<\|"\|>/g, '"') - .replace(/<\|'\\?\|>/g, "'"); - argsBody = argsBody.replace(/(?<=^|,)\s*(\w+)\s*:/g, '"$1":'); - try { - const args = JSON.parse(`{${argsBody}}`); - results.push({ name: toolName, arguments: args }); - } catch { - results.push({ name: toolName, arguments: {} }); - } - } - } - - // Convert to OpenAI tool_calls format. - return results.map((obj, i) => ({ - id: `fallback_call_${Date.now()}_${i}`, - type: 'function', - function: { - name: obj.name, - arguments: typeof obj.arguments === 'string' - ? obj.arguments - : JSON.stringify(obj.arguments || obj.parameters || {}), - }, - })); + return parseToolCallsFromText(text, allowedNames); } // ───────────────────────────────────────────────────────────────────── diff --git a/src/chrome/src/agent/tool-call-parser.js b/src/chrome/src/agent/tool-call-parser.js new file mode 100644 index 000000000..17a5da201 --- /dev/null +++ b/src/chrome/src/agent/tool-call-parser.js @@ -0,0 +1,123 @@ +// Browser-free fallback parser for local models that emit tool calls as text +// instead of using the provider's structured tool_calls field. This file is +// mirrored in the Firefox tree; keep both copies byte-identical. + +/** + * Parse common text tool-call formats into OpenAI-style tool call objects. + * Only names in allowedNames are accepted. + */ +export function parseToolCallsFromText(text, allowedNames) { + if (!text || text.length > 10000) return []; + + const results = []; + const parseXmlParamValue = (value) => { + const cleaned = String(value || '') + .replace(/<[^>]+>/g, '') + .trim(); + if (!cleaned) return ''; + try { + if (/^(?:"|'.*'|\{|\[|-?\d|true\b|false\b|null\b)/i.test(cleaned)) { + return JSON.parse(cleaned.replace(/^'([\s\S]*)'$/, '"$1"')); + } + } catch { /* fall through to string cleanup */ } + return cleaned.replace(/^["']+|["']+$/g, ''); + }; + + const patterns = [ + /\s*([\s\S]*?)\s*<\/tool_call>/gi, + /<\|tool_call\|?>\s*([\s\S]*?)\s*<\|?\/?tool_call\|?>/gi, + /\s*([\s\S]*?)\s*<\/functioncall>/gi, + ]; + + for (const re of patterns) { + let match; + while ((match = re.exec(text)) !== null) { + const inner = match[1].trim(); + try { + const obj = JSON.parse(inner); + if (obj && obj.name && allowedNames.has(obj.name)) { + results.push(obj); + continue; + } + } catch { /* not JSON — try call:name{} format below */ } + + const callMatch = /^call:(\w+)\s*\{([\s\S]*)\}$/.exec(inner); + if (callMatch && allowedNames.has(callMatch[1])) { + const toolName = callMatch[1]; + let argsBody = callMatch[2] + .replace(/<\|"\|>/g, '"') + .replace(/<\|'\\?\|>/g, "'"); + argsBody = argsBody.replace(/(?<=^|,)\s*(\w+)\s*:/g, '"$1":'); + try { + const args = JSON.parse(`{${argsBody}}`); + results.push({ name: toolName, arguments: args }); + } catch { + results.push({ name: toolName, arguments: {} }); + } + } + } + } + + // XML-ish tool-call format used by some local/chat-template models: + // ref_6... + const xmlToolRe = /\s*\s*([\s\S]*?)\s*<\/function>\s*<\/tool_call>/gi; + let xmlMatch; + while ((xmlMatch = xmlToolRe.exec(text)) !== null) { + const toolName = xmlMatch[1] || xmlMatch[2]; + if (!allowedNames.has(toolName)) continue; + const body = xmlMatch[3] || ''; + const args = {}; + const paramRe = /\s*([\s\S]*?)\s*<\/parameter>/gi; + let paramMatch; + while ((paramMatch = paramRe.exec(body)) !== null) { + const key = paramMatch[1] || paramMatch[2]; + if (!key) continue; + args[key] = parseXmlParamValue(paramMatch[3]); + } + results.push({ name: toolName, arguments: args }); + } + + if (results.length === 0) { + const bareRe = /\{[^{}]*"name"\s*:\s*"(\w+)"[^{}]*\}/g; + let match; + while ((match = bareRe.exec(text)) !== null) { + if (!allowedNames.has(match[1])) continue; + try { + const obj = JSON.parse(match[0]); + if (obj && obj.name && allowedNames.has(obj.name)) { + results.push(obj); + } + } catch { /* skip */ } + } + } + + if (results.length === 0) { + const callRe = /call:(\w+)\s*\{([\s\S]*?)\}/g; + let match; + while ((match = callRe.exec(text)) !== null) { + if (!allowedNames.has(match[1])) continue; + const toolName = match[1]; + let argsBody = match[2] + .replace(/<\|"\|>/g, '"') + .replace(/<\|'\\?\|>/g, "'"); + argsBody = argsBody.replace(/(?<=^|,)\s*(\w+)\s*:/g, '"$1":'); + try { + const args = JSON.parse(`{${argsBody}}`); + results.push({ name: toolName, arguments: args }); + } catch { + results.push({ name: toolName, arguments: {} }); + } + } + } + + return results.map((obj, index) => ({ + id: `fallback_call_${Date.now()}_${index}`, + type: 'function', + function: { + name: obj.name, + arguments: typeof obj.arguments === 'string' + ? obj.arguments + : JSON.stringify(obj.arguments || obj.parameters || {}), + }, + })); +} diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index 93827d3b9..0cee0505c 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -1,6 +1,7 @@ import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, getToolsForMode, SYSTEM_PROMPT_ASK, SYSTEM_PROMPT_ACT, SYSTEM_PROMPT_ACT_COMPACT, SYSTEM_PROMPT_ACT_MID, SYSTEM_PROMPT_DEV_APPENDIX } from './tools.js'; import { handleDoneJson } from './cloud-output.js'; import { LoopDetector } from './loop-detector.js'; +import { parseToolCallsFromText } from './tool-call-parser.js'; import { BROWSER_MUTATION_TOOLS, STATE_CHANGE_TOOLS as SHARED_STATE_CHANGE_TOOLS } from './mutation-tools.js'; import { isCredentialField, CREDENTIAL_NOTE_STRICT, STRICT_SECRET_SYSTEM_NOTE } from './credential-fields.js'; import { detectProgressAction, formatLedgerRow, formatLedgerSummary, isBlockedLedgerDowngrade, isTerminalLedgerStatus, isValidLedgerStatus, ledgerDoneBlock, ledgerRowKey, normalizeLedgerStatus, progressCounts, selectLedgerRows, unresolvedLedgerRows, upsertLedgerItems } from './progress-ledger.js'; @@ -14013,124 +14014,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d * was found. Only tool names present in the allowed-name set are accepted. */ _tryParseToolCallsFromText(text, allowedNames = AGENT_TOOL_NAMES) { - if (!text || text.length > 10000) return []; - - const results = []; - const parseXmlParamValue = (value) => { - const cleaned = String(value || '') - .replace(/<[^>]+>/g, '') - .trim(); - if (!cleaned) return ''; - try { - if (/^(?:"|'.*'|\{|\[|-?\d|true\b|false\b|null\b)/i.test(cleaned)) { - return JSON.parse(cleaned.replace(/^'([\s\S]*)'$/, '"$1"')); - } - } catch { /* fall through to string cleanup */ } - return cleaned.replace(/^["']+|["']+$/g, ''); - }; - - const patterns = [ - /\s*([\s\S]*?)\s*<\/tool_call>/gi, - /<\|tool_call\|?>\s*([\s\S]*?)\s*<\|?\/?tool_call\|?>/gi, - /\s*([\s\S]*?)\s*<\/functioncall>/gi, - ]; - - for (const re of patterns) { - let m; - while ((m = re.exec(text)) !== null) { - const inner = m[1].trim(); - // Try JSON first (most common). - try { - const obj = JSON.parse(inner); - if (obj && obj.name && allowedNames.has(obj.name)) { - results.push(obj); - continue; - } - } catch { /* not JSON — try call:name{} format below */ } - - // call:toolName{key:<|"|>value<|"|>, ...} format. - const callMatch = /^call:(\w+)\s*\{([\s\S]*)\}$/.exec(inner); - if (callMatch && allowedNames.has(callMatch[1])) { - const toolName = callMatch[1]; - let argsBody = callMatch[2] - .replace(/<\|"\|>/g, '"') - .replace(/<\|'\\?\|>/g, "'"); - argsBody = argsBody.replace(/(?<=^|,)\s*(\w+)\s*:/g, '"$1":'); - try { - const args = JSON.parse(`{${argsBody}}`); - results.push({ name: toolName, arguments: args }); - } catch { - results.push({ name: toolName, arguments: {} }); - } - continue; - } - } - } - - // XML-ish tool-call format used by some local/chat-template models: - // ref_6... - const xmlToolRe = /\s*\s*([\s\S]*?)\s*<\/function>\s*<\/tool_call>/gi; - let xmlMatch; - while ((xmlMatch = xmlToolRe.exec(text)) !== null) { - const toolName = xmlMatch[1] || xmlMatch[2]; - if (!allowedNames.has(toolName)) continue; - const body = xmlMatch[3] || ''; - const args = {}; - const paramRe = /\s*([\s\S]*?)\s*<\/parameter>/gi; - let paramMatch; - while ((paramMatch = paramRe.exec(body)) !== null) { - const key = paramMatch[1] || paramMatch[2]; - if (!key) continue; - args[key] = parseXmlParamValue(paramMatch[3]); - } - results.push({ name: toolName, arguments: args }); - } - - // Fallback: scan for bare JSON objects containing a "name" key. - if (results.length === 0) { - const bareRe = /\{[^{}]*"name"\s*:\s*"(\w+)"[^{}]*\}/g; - let m; - while ((m = bareRe.exec(text)) !== null) { - if (!allowedNames.has(m[1])) continue; - try { - const obj = JSON.parse(m[0]); - if (obj && obj.name && allowedNames.has(obj.name)) { - results.push(obj); - } - } catch { /* skip */ } - } - } - - // Last resort: call:toolName{...} outside of any wrapper tags. - if (results.length === 0) { - const callRe = /call:(\w+)\s*\{([\s\S]*?)\}/g; - let m; - while ((m = callRe.exec(text)) !== null) { - if (!allowedNames.has(m[1])) continue; - const toolName = m[1]; - let argsBody = m[2] - .replace(/<\|"\|>/g, '"') - .replace(/<\|'\\?\|>/g, "'"); - argsBody = argsBody.replace(/(?<=^|,)\s*(\w+)\s*:/g, '"$1":'); - try { - const args = JSON.parse(`{${argsBody}}`); - results.push({ name: toolName, arguments: args }); - } catch { - results.push({ name: toolName, arguments: {} }); - } - } - } - - return results.map((obj, i) => ({ - id: `fallback_call_${Date.now()}_${i}`, - type: 'function', - function: { - name: obj.name, - arguments: typeof obj.arguments === 'string' - ? obj.arguments - : JSON.stringify(obj.arguments || obj.parameters || {}), - }, - })); + return parseToolCallsFromText(text, allowedNames); } // ───────────────────────────────────────────────────────────────────── diff --git a/src/firefox/src/agent/tool-call-parser.js b/src/firefox/src/agent/tool-call-parser.js new file mode 100644 index 000000000..17a5da201 --- /dev/null +++ b/src/firefox/src/agent/tool-call-parser.js @@ -0,0 +1,123 @@ +// Browser-free fallback parser for local models that emit tool calls as text +// instead of using the provider's structured tool_calls field. This file is +// mirrored in the Firefox tree; keep both copies byte-identical. + +/** + * Parse common text tool-call formats into OpenAI-style tool call objects. + * Only names in allowedNames are accepted. + */ +export function parseToolCallsFromText(text, allowedNames) { + if (!text || text.length > 10000) return []; + + const results = []; + const parseXmlParamValue = (value) => { + const cleaned = String(value || '') + .replace(/<[^>]+>/g, '') + .trim(); + if (!cleaned) return ''; + try { + if (/^(?:"|'.*'|\{|\[|-?\d|true\b|false\b|null\b)/i.test(cleaned)) { + return JSON.parse(cleaned.replace(/^'([\s\S]*)'$/, '"$1"')); + } + } catch { /* fall through to string cleanup */ } + return cleaned.replace(/^["']+|["']+$/g, ''); + }; + + const patterns = [ + /\s*([\s\S]*?)\s*<\/tool_call>/gi, + /<\|tool_call\|?>\s*([\s\S]*?)\s*<\|?\/?tool_call\|?>/gi, + /\s*([\s\S]*?)\s*<\/functioncall>/gi, + ]; + + for (const re of patterns) { + let match; + while ((match = re.exec(text)) !== null) { + const inner = match[1].trim(); + try { + const obj = JSON.parse(inner); + if (obj && obj.name && allowedNames.has(obj.name)) { + results.push(obj); + continue; + } + } catch { /* not JSON — try call:name{} format below */ } + + const callMatch = /^call:(\w+)\s*\{([\s\S]*)\}$/.exec(inner); + if (callMatch && allowedNames.has(callMatch[1])) { + const toolName = callMatch[1]; + let argsBody = callMatch[2] + .replace(/<\|"\|>/g, '"') + .replace(/<\|'\\?\|>/g, "'"); + argsBody = argsBody.replace(/(?<=^|,)\s*(\w+)\s*:/g, '"$1":'); + try { + const args = JSON.parse(`{${argsBody}}`); + results.push({ name: toolName, arguments: args }); + } catch { + results.push({ name: toolName, arguments: {} }); + } + } + } + } + + // XML-ish tool-call format used by some local/chat-template models: + // ref_6... + const xmlToolRe = /\s*\s*([\s\S]*?)\s*<\/function>\s*<\/tool_call>/gi; + let xmlMatch; + while ((xmlMatch = xmlToolRe.exec(text)) !== null) { + const toolName = xmlMatch[1] || xmlMatch[2]; + if (!allowedNames.has(toolName)) continue; + const body = xmlMatch[3] || ''; + const args = {}; + const paramRe = /\s*([\s\S]*?)\s*<\/parameter>/gi; + let paramMatch; + while ((paramMatch = paramRe.exec(body)) !== null) { + const key = paramMatch[1] || paramMatch[2]; + if (!key) continue; + args[key] = parseXmlParamValue(paramMatch[3]); + } + results.push({ name: toolName, arguments: args }); + } + + if (results.length === 0) { + const bareRe = /\{[^{}]*"name"\s*:\s*"(\w+)"[^{}]*\}/g; + let match; + while ((match = bareRe.exec(text)) !== null) { + if (!allowedNames.has(match[1])) continue; + try { + const obj = JSON.parse(match[0]); + if (obj && obj.name && allowedNames.has(obj.name)) { + results.push(obj); + } + } catch { /* skip */ } + } + } + + if (results.length === 0) { + const callRe = /call:(\w+)\s*\{([\s\S]*?)\}/g; + let match; + while ((match = callRe.exec(text)) !== null) { + if (!allowedNames.has(match[1])) continue; + const toolName = match[1]; + let argsBody = match[2] + .replace(/<\|"\|>/g, '"') + .replace(/<\|'\\?\|>/g, "'"); + argsBody = argsBody.replace(/(?<=^|,)\s*(\w+)\s*:/g, '"$1":'); + try { + const args = JSON.parse(`{${argsBody}}`); + results.push({ name: toolName, arguments: args }); + } catch { + results.push({ name: toolName, arguments: {} }); + } + } + } + + return results.map((obj, index) => ({ + id: `fallback_call_${Date.now()}_${index}`, + type: 'function', + function: { + name: obj.name, + arguments: typeof obj.arguments === 'string' + ? obj.arguments + : JSON.stringify(obj.arguments || obj.parameters || {}), + }, + })); +} diff --git a/test/run.js b/test/run.js index 8d284e3c0..63578fcbd 100644 --- a/test/run.js +++ b/test/run.js @@ -442,6 +442,12 @@ const CaptchaGateCh = await import( const CaptchaGateFx = await import( 'file://' + path.join(ROOT, 'src/firefox/src/agent/captcha-gate.js').replace(/\\/g, '/') ); +const ToolCallParserCh = await import( + 'file://' + path.join(ROOT, 'src/chrome/src/agent/tool-call-parser.js').replace(/\\/g, '/') +); +const ToolCallParserFx = await import( + 'file://' + path.join(ROOT, 'src/firefox/src/agent/tool-call-parser.js').replace(/\\/g, '/') +); // The mutating-tool surface differs per build, so it lives outside the // byte-identical loop-detector module. Import the real sets rather than // restating them here — a hand-copied list is exactly the drift this suite @@ -43802,6 +43808,103 @@ test('streamed plain final answers cannot bypass unresolved progress rows', asyn } }); +test('text tool-call parser is production code with format and allowlist coverage', () => { + assert.equal( + fs.readFileSync(path.join(ROOT, 'src/chrome/src/agent/tool-call-parser.js'), 'utf8'), + fs.readFileSync(path.join(ROOT, 'src/firefox/src/agent/tool-call-parser.js'), 'utf8'), + 'chrome and firefox tool-call parsers must remain byte-identical', + ); + const allowed = new Set(['click', 'click_ax', 'navigate', 'read_page']); + const cases = [ + { + label: 'tool_call JSON with nested arguments', + raw: '{"name":"click","arguments":{"text":"Save","meta":{"source":"dialog"}}}', + expected: [{ name: 'click', args: { text: 'Save', meta: { source: 'dialog' } } }], + }, + { + label: 'token wrapper', + raw: '<|tool_call|>{"name":"read_page","arguments":{}}<|/tool_call|>', + expected: [{ name: 'read_page', args: {} }], + }, + { + label: 'functioncall wrapper', + raw: '{"name":"navigate","arguments":{"url":"https://example.test/path"}}', + expected: [{ name: 'navigate', args: { url: 'https://example.test/path' } }], + }, + { + label: 'custom quote tokens', + raw: 'call:click{text:<|"|>Save<|"|>}', + expected: [{ name: 'click', args: { text: 'Save' } }], + }, + { + label: 'bare JSON with string arguments', + raw: '{"name":"read_page","arguments":"[]"}', + expected: [{ name: 'read_page', args: [] }], + }, + { + label: 'XML typed parameters', + raw: [ + '', + '"Save"', + '2', + 'true', + '[10,20]', + '', + ].join(''), + expected: [{ + name: 'click', + args: { name: 'Save', index: 2, force: true, coordinates: [10, 20] }, + }], + }, + { + label: 'multiple wrappers preserve order', + raw: [ + '{"name":"read_page","arguments":{}}', + '{"name":"click_ax","arguments":{"ref_id":"ref_7"}}', + ].join('\n'), + expected: [ + { name: 'read_page', args: {} }, + { name: 'click_ax', args: { ref_id: 'ref_7' } }, + ], + }, + ]; + + for (const parser of [ToolCallParserCh, ToolCallParserFx]) { + for (const scenario of cases) { + const calls = parser.parseToolCallsFromText(scenario.raw, allowed); + assert.deepEqual( + calls.map(call => ({ + name: call.function.name, + args: JSON.parse(call.function.arguments), + })), + scenario.expected, + scenario.label, + ); + assert.equal( + calls.every((call, index) => + new RegExp(`^fallback_call_\\d+_${index}$`).test(call.id) + ), + true, + `${scenario.label}: fallback IDs were not stable OpenAI-style call IDs`, + ); + } + + assert.deepEqual( + parser.parseToolCallsFromText( + '{"name":"execute_js","arguments":{"code":"alert(1)"}}', + allowed, + ), + [], + 'disallowed tool name bypassed the allowlist', + ); + assert.deepEqual( + parser.parseToolCallsFromText('x'.repeat(10001), allowed), + [], + 'oversized model text was parsed', + ); + } +}); + test('agent parses XML-style raw tool calls with ref_id parameters', () => { const raw = [ '', From b52e7d9d2c0452d3cdb16a70236eddfa09878ddd Mon Sep 17 00:00:00 2001 From: alectimison-maker Date: Sun, 26 Jul 2026 15:02:06 +0800 Subject: [PATCH 3/8] fix(firefox): block rapid duplicate submits --- src/chrome/ARCHITECTURE.md | 2 +- src/chrome/src/agent/agent.js | 46 ++--- src/chrome/src/agent/submit-click-guard.js | 52 ++++++ src/firefox/ARCHITECTURE.md | 13 +- src/firefox/src/agent/agent.js | 14 ++ src/firefox/src/agent/submit-click-guard.js | 52 ++++++ test/run.js | 177 ++++++++++++++++++++ 7 files changed, 316 insertions(+), 40 deletions(-) create mode 100644 src/chrome/src/agent/submit-click-guard.js create mode 100644 src/firefox/src/agent/submit-click-guard.js diff --git a/src/chrome/ARCHITECTURE.md b/src/chrome/ARCHITECTURE.md index cf43a56da..4b9649b6c 100644 --- a/src/chrome/ARCHITECTURE.md +++ b/src/chrome/ARCHITECTURE.md @@ -419,7 +419,7 @@ System prompt has a new "MODALS & DIALOGS" section that describes the intended f ### Duplicate-submit guard (v3.6.5+) -Before any `click` whose resolved text matches `^(create|save|submit|add|post|publish|send|confirm|place order|pay|checkout|update|finish|done)\b` the agent checks a per-tab+URL 45-second window. Duplicate clicks in that window are blocked unless `_allowResubmit` is set. Prevents the "clicked Create three times → three products created" failure mode. +Before any submit-like text `click`, the agent checks a per-tab+URL 45-second window. Duplicate clicks in that window are blocked unless `_allowResubmit` is set. The browser-free guard is byte-identical in Chrome and Firefox, preventing the "clicked Create three times → three products created" failure mode in both builds. ### API shortcut observer (v18.0.0) diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index d7c39a485..9c38e64be 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -2,6 +2,7 @@ import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, getToolsForMo import { handleDoneJson } from './cloud-output.js'; import { LoopDetector } from './loop-detector.js'; import { BROWSER_MUTATION_TOOLS, STATE_CHANGE_TOOLS as SHARED_STATE_CHANGE_TOOLS } from './mutation-tools.js'; +import { guardRecentSubmitClick } from './submit-click-guard.js'; import { isCredentialField, CREDENTIAL_NOTE_STRICT, STRICT_SECRET_SYSTEM_NOTE } from './credential-fields.js'; import { detectProgressAction, formatLedgerRow, formatLedgerSummary, isBlockedLedgerDowngrade, isTerminalLedgerStatus, isValidLedgerStatus, ledgerDoneBlock, ledgerRowKey, normalizeLedgerStatus, progressCounts, selectLedgerRows, unresolvedLedgerRows, upsertLedgerItems } from './progress-ledger.js'; import { buildGithubStargazerProgressItems } from './observers/github-stargazers.js'; @@ -15856,41 +15857,16 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d await cdpClient.attach(tabId); - // ── Duplicate submit-click guard ──────────────────────────────── - // The model often mistakes the modal-open link and the in-modal - // submit button on Stripe-style UIs (both labeled "Create product", - // "Add product", etc.) — clicking twice creates duplicate records. - // Track clicks whose text matches a submit-like pattern and block - // a second one on the same tab+URL within a short window, UNLESS - // the URL has changed (real navigation) or the model explicitly - // acknowledges the duplicate via args._allowResubmit = true. - if (args.text && !args._allowResubmit) { - const rawText = String(args.text).trim(); - const submitLikeRE = /^(create|save|submit|add|post|publish|send|confirm|sign up|sign in|log in|register|place order|pay|checkout|update|apply|finish|done)\b/i; - if (submitLikeRE.test(rawText)) { - let curUrl = ''; - try { const t = await chrome.tabs.get(tabId); curUrl = t?.url || ''; } catch (e) {} - const buf = this._recentSubmitClicks.get(tabId) || []; - const key = `${rawText.toLowerCase()}|${curUrl}`; - const now = Date.now(); - // Keep entries from the last 45 seconds - const fresh = buf.filter(e => now - e.ts < 45000); - const match = fresh.find(e => e.key === key); - if (match) { - return { - success: false, - dispatched: false, - blockedDuplicateSubmit: true, - error: `Blocked: you already clicked "${rawText}" on this page ${Math.round((now - match.ts) / 1000)}s ago and the URL has not changed since. Stripe-style UIs often reuse the same label for the modal-OPEN button and the SUBMIT button inside the modal — a second click typically creates a duplicate record. Before clicking "${rawText}" again, verify: (a) that all required fields are actually filled by reading the form/page, (b) that this click is intended as a FIRST submit and not a retry. If the previous click did nothing because a field was empty, fill the field first. If you genuinely need to retry, pass _allowResubmit: true in the args.`, - previousClickUrl: match.url, - currentUrl: curUrl, - secondsSincePrevious: Math.round((now - match.ts) / 1000), - }; - } - fresh.push({ key, ts: now, url: curUrl, text: rawText }); - this._recentSubmitClicks.set(tabId, fresh); - } - } + const duplicateSubmit = await guardRecentSubmitClick( + this._recentSubmitClicks, + tabId, + args, + async () => { + const tab = await chrome.tabs.get(tabId); + return tab?.url || ''; + }, + ); + if (duplicateSubmit) return duplicateSubmit; // ── Global SELECT guard ───────────────────────────────────────── // Inject a capture-phase mousedown+click listener that prevents diff --git a/src/chrome/src/agent/submit-click-guard.js b/src/chrome/src/agent/submit-click-guard.js new file mode 100644 index 000000000..96b5cab8f --- /dev/null +++ b/src/chrome/src/agent/submit-click-guard.js @@ -0,0 +1,52 @@ +// Browser-free duplicate-submit guard. This file is mirrored in the Firefox +// tree; keep both copies byte-identical. + +export const SUBMIT_CLICK_WINDOW_MS = 45_000; + +const SUBMIT_LIKE_CLICK_RE = /^(create|save|submit|add|post|publish|send|confirm|sign up|sign in|log in|register|place order|pay|checkout|update|apply|finish|done)\b/i; + +/** + * Record the first submit-like text click and reject a rapid duplicate on the + * same tab and URL. Returns the tool result to emit when blocked, otherwise + * null. The URL callback is lazy so ordinary clicks and explicit retries do + * not perform a tab lookup. + */ +export async function guardRecentSubmitClick( + recentSubmitClicks, + tabId, + args, + getCurrentUrl, + now = Date.now, +) { + if (!args?.text || args._allowResubmit) return null; + + const rawText = String(args.text).trim(); + if (!SUBMIT_LIKE_CLICK_RE.test(rawText)) return null; + + let currentUrl = ''; + try { + currentUrl = await getCurrentUrl() || ''; + } catch { /* preserve empty-URL fallback */ } + + const entries = recentSubmitClicks.get(tabId) || []; + const key = `${rawText.toLowerCase()}|${currentUrl}`; + const timestamp = now(); + const fresh = entries.filter(entry => timestamp - entry.ts < SUBMIT_CLICK_WINDOW_MS); + const match = fresh.find(entry => entry.key === key); + + if (match) { + return { + success: false, + dispatched: false, + blockedDuplicateSubmit: true, + error: `Blocked: you already clicked "${rawText}" on this page ${Math.round((timestamp - match.ts) / 1000)}s ago and the URL has not changed since. Stripe-style UIs often reuse the same label for the modal-OPEN button and the SUBMIT button inside the modal — a second click typically creates a duplicate record. Before clicking "${rawText}" again, verify: (a) that all required fields are actually filled by reading the form/page, (b) that this click is intended as a FIRST submit and not a retry. If the previous click did nothing because a field was empty, fill the field first. If you genuinely need to retry, pass _allowResubmit: true in the args.`, + previousClickUrl: match.url, + currentUrl, + secondsSincePrevious: Math.round((timestamp - match.ts) / 1000), + }; + } + + fresh.push({ key, ts: timestamp, url: currentUrl, text: rawText }); + recentSubmitClicks.set(tabId, fresh); + return null; +} diff --git a/src/firefox/ARCHITECTURE.md b/src/firefox/ARCHITECTURE.md index b40b7300f..0a113f66d 100644 --- a/src/firefox/ARCHITECTURE.md +++ b/src/firefox/ARCHITECTURE.md @@ -10,7 +10,6 @@ Firefox uses Manifest V2 (background page, not service worker) and has **no acce - **No pixel-perfect / full-page screenshots** — uses `browser.tabs.captureVisibleTab()` instead of CDP `Page.captureScreenshot`. - **No shadow DOM piercing** — content script can read open shadow roots via `element.shadowRoot`, but cannot pierce closed roots. - **No offscreen document** — no HTTP fetch proxy for localhost LLM servers with Private Network Access / CORS issues. User must ensure their local LLM server sends permissive CORS headers. -- **No duplicate-submit guard** — the per-tab submit-throttle (Chrome v3.6.5+) is still Chrome-only. Firefox's agent loop does not block rapid duplicate Create/Submit clicks. `blockedDone` and the ambiguous-click candidate payload were ported to Firefox in v4.0.1 (see "Overlay defenses" below). - **Some Chrome-only tools/features remain absent** — no CDP full-page screenshot, CDP upload automation, tab recording, offscreen fetch proxy, Chrome-only `shadow_dom_query`, or closed-shadow-root traversal. Everything else — the agent loop, LLM providers, site adapters, Ask/Act/Dev mode routing, Plan before Act, loop detection, API shortcut observer, trace recorder, scheduler, context management — is architecturally identical to Chrome unless noted below. @@ -172,10 +171,9 @@ Firefox's AX tools use synthetic events only — there is no trusted-event path. ### What was intentionally skipped in the Firefox port -These Chrome v3.6.x features depend on CDP or agent-level state and were not ported — they can be re-evaluated later: +These Chrome v3.6.x features depend on CDP and were not ported — they can be re-evaluated later: - **CDP-enriched `click_ax` frontmost resolution** — when `click_ax` lands on a node that overlaps many candidates, Chrome re-queries via CDP to pick the frontmost hit. Firefox relies on the initial ref_id resolution plus the v4.0.1 occlusion hit-test (see below). -- **Duplicate-submit guard** — Chrome's agent.js tracks recent submit tool calls and blocks duplicates within a short window. Not in Firefox's agent loop. - **Offscreen fetch fallback** — Chrome falls through to an offscreen-document proxy when direct fetch fails (common for localhost LLM servers and private-network destinations). Firefox has no offscreen API; local servers must handle CORS themselves. ### Overlay defenses (v4.0.1+) @@ -190,6 +188,14 @@ Brought to Chrome parity in v4.0.1 — same three layers, synthetic-event-safe s System prompt has a new "MODALS & DIALOGS" section describing the intended flow and the "dialog still open" failure pattern. +### Duplicate-submit guard + +Submit-like text clicks use the same browser-free guard as Chrome. The first +click is recorded by tab, normalized label, and current URL; another matching +click within 45 seconds is blocked unless `_allowResubmit` explicitly +acknowledges the retry. Navigation, expired entries, ordinary labels, and +validation-rejected submits remain eligible for a fresh click. + --- ## Skills and Dynamic Skill Tools @@ -546,7 +552,6 @@ when the tab conversation already has `/allow-api`. | No full-page screenshot | Only visible viewport | Scroll + multiple captures | | No shadow-root piercing (closed) | Can't read closed shadow roots | Dev-mode `execute_js` with manual traversal | | No arbitrary-path/CDP upload | Cannot attach an arbitrary local path silently | Use a prior `downloadId` re-fetch or WebBrain's user file picker | -| No duplicate-submit guard | Agent may submit twice if LLM loops | Rely on site-level idempotence / user watches | | No ambiguous-click CDP enrichment | Overlapping hit-target ambiguity resolved by ref_id only | Prompting / adapter guidance | | MV2 background page | Less efficient than MV3 service worker | `persistent: false` helps | diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index 93827d3b9..68cfae79e 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -2,6 +2,7 @@ import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, getToolsForMo import { handleDoneJson } from './cloud-output.js'; import { LoopDetector } from './loop-detector.js'; import { BROWSER_MUTATION_TOOLS, STATE_CHANGE_TOOLS as SHARED_STATE_CHANGE_TOOLS } from './mutation-tools.js'; +import { guardRecentSubmitClick } from './submit-click-guard.js'; import { isCredentialField, CREDENTIAL_NOTE_STRICT, STRICT_SECRET_SYSTEM_NOTE } from './credential-fields.js'; import { detectProgressAction, formatLedgerRow, formatLedgerSummary, isBlockedLedgerDowngrade, isTerminalLedgerStatus, isValidLedgerStatus, ledgerDoneBlock, ledgerRowKey, normalizeLedgerStatus, progressCounts, selectLedgerRows, unresolvedLedgerRows, upsertLedgerItems } from './progress-ledger.js'; import { buildGithubStargazerProgressItems } from './observers/github-stargazers.js'; @@ -13848,6 +13849,19 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d } } catch { /* tab lookup failures are non-fatal — fall through */ } + if (name === 'click') { + const duplicateSubmit = await guardRecentSubmitClick( + this._recentSubmitClicks, + tabId, + args, + async () => { + const tab = await browser.tabs.get(tabId); + return tab?.url || ''; + }, + ); + if (duplicateSubmit) return duplicateSubmit; + } + const axScope = this._lastAxScopes.get(tabId); const contentArgs = (name === 'click_ax' || name === 'set_checked') && axScope?.documentToken ? { diff --git a/src/firefox/src/agent/submit-click-guard.js b/src/firefox/src/agent/submit-click-guard.js new file mode 100644 index 000000000..96b5cab8f --- /dev/null +++ b/src/firefox/src/agent/submit-click-guard.js @@ -0,0 +1,52 @@ +// Browser-free duplicate-submit guard. This file is mirrored in the Firefox +// tree; keep both copies byte-identical. + +export const SUBMIT_CLICK_WINDOW_MS = 45_000; + +const SUBMIT_LIKE_CLICK_RE = /^(create|save|submit|add|post|publish|send|confirm|sign up|sign in|log in|register|place order|pay|checkout|update|apply|finish|done)\b/i; + +/** + * Record the first submit-like text click and reject a rapid duplicate on the + * same tab and URL. Returns the tool result to emit when blocked, otherwise + * null. The URL callback is lazy so ordinary clicks and explicit retries do + * not perform a tab lookup. + */ +export async function guardRecentSubmitClick( + recentSubmitClicks, + tabId, + args, + getCurrentUrl, + now = Date.now, +) { + if (!args?.text || args._allowResubmit) return null; + + const rawText = String(args.text).trim(); + if (!SUBMIT_LIKE_CLICK_RE.test(rawText)) return null; + + let currentUrl = ''; + try { + currentUrl = await getCurrentUrl() || ''; + } catch { /* preserve empty-URL fallback */ } + + const entries = recentSubmitClicks.get(tabId) || []; + const key = `${rawText.toLowerCase()}|${currentUrl}`; + const timestamp = now(); + const fresh = entries.filter(entry => timestamp - entry.ts < SUBMIT_CLICK_WINDOW_MS); + const match = fresh.find(entry => entry.key === key); + + if (match) { + return { + success: false, + dispatched: false, + blockedDuplicateSubmit: true, + error: `Blocked: you already clicked "${rawText}" on this page ${Math.round((timestamp - match.ts) / 1000)}s ago and the URL has not changed since. Stripe-style UIs often reuse the same label for the modal-OPEN button and the SUBMIT button inside the modal — a second click typically creates a duplicate record. Before clicking "${rawText}" again, verify: (a) that all required fields are actually filled by reading the form/page, (b) that this click is intended as a FIRST submit and not a retry. If the previous click did nothing because a field was empty, fill the field first. If you genuinely need to retry, pass _allowResubmit: true in the args.`, + previousClickUrl: match.url, + currentUrl, + secondsSincePrevious: Math.round((timestamp - match.ts) / 1000), + }; + } + + fresh.push({ key, ts: timestamp, url: currentUrl, text: rawText }); + recentSubmitClicks.set(tabId, fresh); + return null; +} diff --git a/test/run.js b/test/run.js index 8d284e3c0..28e5fcdab 100644 --- a/test/run.js +++ b/test/run.js @@ -442,6 +442,12 @@ const CaptchaGateCh = await import( const CaptchaGateFx = await import( 'file://' + path.join(ROOT, 'src/firefox/src/agent/captcha-gate.js').replace(/\\/g, '/') ); +const SubmitClickGuardCh = await import( + 'file://' + path.join(ROOT, 'src/chrome/src/agent/submit-click-guard.js').replace(/\\/g, '/') +); +const SubmitClickGuardFx = await import( + 'file://' + path.join(ROOT, 'src/firefox/src/agent/submit-click-guard.js').replace(/\\/g, '/') +); // The mutating-tool surface differs per build, so it lives outside the // byte-identical loop-detector module. Import the real sets rather than // restating them here — a hand-copied list is exactly the drift this suite @@ -34004,6 +34010,177 @@ test('execute_js submissions receive form validation feedback', async () => { } }); +test('duplicate submit-click guard is production code with Chrome/Firefox parity', async () => { + assert.equal( + fs.readFileSync(path.join(ROOT, 'src/chrome/src/agent/submit-click-guard.js'), 'utf8'), + fs.readFileSync(path.join(ROOT, 'src/firefox/src/agent/submit-click-guard.js'), 'utf8'), + 'chrome and firefox duplicate-submit guards must remain byte-identical', + ); + + for (const guardModule of [SubmitClickGuardCh, SubmitClickGuardFx]) { + const recentClicks = new Map(); + const tabId = 6201; + let now = 100_000; + let currentUrl = 'https://example.test/products'; + let urlReads = 0; + const getCurrentUrl = async () => { + urlReads += 1; + return currentUrl; + }; + + assert.equal( + await guardModule.guardRecentSubmitClick( + recentClicks, + tabId, + { text: ' Save changes ' }, + getCurrentUrl, + () => now, + ), + null, + 'first submit-like click should be recorded and allowed', + ); + assert.deepEqual(recentClicks.get(tabId), [{ + key: 'save changes|https://example.test/products', + ts: 100_000, + url: 'https://example.test/products', + text: 'Save changes', + }]); + + now += 10_400; + const duplicate = await guardModule.guardRecentSubmitClick( + recentClicks, + tabId, + { text: 'SAVE changes' }, + getCurrentUrl, + () => now, + ); + assert.equal(duplicate.success, false); + assert.equal(duplicate.dispatched, false); + assert.equal(duplicate.blockedDuplicateSubmit, true); + assert.equal(duplicate.previousClickUrl, 'https://example.test/products'); + assert.equal(duplicate.currentUrl, 'https://example.test/products'); + assert.equal(duplicate.secondsSincePrevious, 10); + assert.match(duplicate.error, /pass _allowResubmit: true/i); + + currentUrl = 'https://example.test/products/new'; + assert.equal( + await guardModule.guardRecentSubmitClick( + recentClicks, + tabId, + { text: 'Save changes' }, + getCurrentUrl, + () => now, + ), + null, + 'the same label after navigation should be allowed', + ); + + now += guardModule.SUBMIT_CLICK_WINDOW_MS; + assert.equal( + await guardModule.guardRecentSubmitClick( + recentClicks, + tabId, + { text: 'Save changes' }, + getCurrentUrl, + () => now, + ), + null, + 'an entry exactly at the expiry boundary should be allowed', + ); + assert.equal(recentClicks.get(tabId).length, 1, 'expired submit-click entries were not pruned'); + + const readsBeforeIgnoredCalls = urlReads; + assert.equal( + await guardModule.guardRecentSubmitClick( + recentClicks, + tabId, + { text: 'Save changes', _allowResubmit: true }, + getCurrentUrl, + () => now, + ), + null, + 'explicit resubmit acknowledgement should bypass the guard', + ); + assert.equal( + await guardModule.guardRecentSubmitClick( + recentClicks, + tabId, + { text: 'Continue' }, + getCurrentUrl, + () => now, + ), + null, + 'non-submit-like labels should bypass the guard', + ); + assert.equal(urlReads, readsBeforeIgnoredCalls, 'bypassed clicks should not query the tab URL'); + + const failedLookupClicks = new Map(); + const failingUrlLookup = async () => { + throw new Error('tab closed'); + }; + assert.equal( + await guardModule.guardRecentSubmitClick( + failedLookupClicks, + tabId, + { text: 'Publish' }, + failingUrlLookup, + () => now, + ), + null, + ); + assert.equal( + ( + await guardModule.guardRecentSubmitClick( + failedLookupClicks, + tabId, + { text: 'Publish' }, + failingUrlLookup, + () => now, + ) + ).blockedDuplicateSubmit, + true, + 'URL lookup failures should retain the existing empty-URL guard behavior', + ); + } +}); + +test('Firefox blocks a duplicate submit click before content-script dispatch', async () => { + const originalBrowser = globalThis.browser; + const tabId = 6202; + const url = 'https://example.test/products'; + let dispatches = 0; + + try { + globalThis.browser = { + tabs: { + get: async () => ({ id: tabId, url }), + sendMessage: async () => { + dispatches += 1; + return { success: true }; + }, + }, + }; + + const agent = new AgentFx({ getVisionProvider: async () => null }); + agent._isPdfTab = async () => false; + agent._recentSubmitClicks.set(tabId, [{ + key: `save|${url}`, + ts: Date.now(), + url, + text: 'Save', + }]); + + const result = await agent.executeTool(tabId, 'click', { text: 'Save' }); + assert.equal(result.success, false); + assert.equal(result.dispatched, false); + assert.equal(result.blockedDuplicateSubmit, true); + assert.equal(dispatches, 0, 'Firefox dispatched a duplicate submit click to the content script'); + } finally { + if (originalBrowser === undefined) delete globalThis.browser; + else globalThis.browser = originalBrowser; + } +}); + test('agent returns form validation messages and blocks unchanged repeat submits', async () => { const url = 'https://addons.mozilla.org/en-US/developers/addon/webbrain/versions/submit/'; const invalidField = { From 300138bb6a6763743ecd390a662e3abb3f0b57ac Mon Sep 17 00:00:00 2001 From: alectimison-maker Date: Sun, 26 Jul 2026 15:28:47 +0800 Subject: [PATCH 4/8] feat: add ATIF trace converter --- docs/export-and-workflow-formats.md | 26 ++ scripts/trace-to-atif.mjs | 354 ++++++++++++++++++++++++++++ test/run.js | 252 ++++++++++++++++++++ 3 files changed, 632 insertions(+) create mode 100644 scripts/trace-to-atif.mjs diff --git a/docs/export-and-workflow-formats.md b/docs/export-and-workflow-formats.md index 2ca265135..2a70574c6 100644 --- a/docs/export-and-workflow-formats.md +++ b/docs/export-and-workflow-formats.md @@ -59,6 +59,32 @@ Screenshot events can include `screenshot_base64` or `screenshot_dataUrl`. Consumers should check `schema`, tolerate additional fields, and avoid relying on undocumented event internals. +### Convert a trace to ATIF v1.7 + +The dependency-free converter turns one Traces-page JSON export into the +[Agent Trajectory Interchange Format (ATIF)](https://github.com/harbor-framework/harbor/blob/main/rfcs/0001-trajectory-format.md): + +```bash +node scripts/trace-to-atif.mjs webbrain-trace-model-run.json +``` + +By default this writes `webbrain-trace-model-run.atif.json` next to the source +file. Pass a second path to choose another destination, or `-` to write the +trajectory to standard output: + +```bash +node scripts/trace-to-atif.mjs trace.json trajectory.json +node scripts/trace-to-atif.mjs trace.json - +``` + +The converter maps user and agent messages, tool calls and observations, token +metrics, errors, model metadata, and final content. It does not upload data. +Screenshots and verbose diagnostic event kinds are counted in +`extra.omitted_event_counts` but are not copied: ATIF represents images as +files referenced alongside the trajectory, while a WebBrain JSON export embeds +them as data URLs or base64. The converted trajectory remains sensitive because +it still contains prompts, tool arguments, URLs, and results. + ## Settings snapshots: `webbrain-config/1` `/export --config` creates a versioned Settings snapshot: diff --git a/scripts/trace-to-atif.mjs b/scripts/trace-to-atif.mjs new file mode 100644 index 000000000..254838ef1 --- /dev/null +++ b/scripts/trace-to-atif.mjs @@ -0,0 +1,354 @@ +#!/usr/bin/env node + +/** + * Convert a Traces-page `webbrain-trace/1` JSON export to ATIF v1.7. + * + * This deliberately stays offline and dependency-free. Screenshots and verbose + * diagnostic events are counted but not copied because ATIF represents images + * as files referenced alongside the trajectory, not embedded data URLs. + */ + +import fs from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +const SOURCE_SCHEMA = 'webbrain-trace/1'; +const ATIF_SCHEMA_VERSION = 'ATIF-v1.7'; + +function isObject(value) { + return !!value && typeof value === 'object' && !Array.isArray(value); +} + +function finiteNumber(value) { + return typeof value === 'number' && Number.isFinite(value) ? value : undefined; +} + +function timestamp(value) { + if (value == null || value === '') return undefined; + const date = new Date(value); + return Number.isNaN(date.getTime()) ? undefined : date.toISOString(); +} + +function jsonSafe(value, fallback) { + try { + const serialized = JSON.stringify(value); + return serialized === undefined ? fallback : JSON.parse(serialized); + } catch { + return fallback; + } +} + +function compactObject(value) { + return Object.fromEntries( + Object.entries(value).filter(([, item]) => item !== undefined), + ); +} + +function parseArguments(value) { + if (isObject(value)) return { arguments: jsonSafe(value, {}) }; + if (typeof value === 'string') { + try { + const parsed = JSON.parse(value); + if (isObject(parsed)) return { arguments: parsed }; + } catch {} + return { + arguments: {}, + extra: { raw_arguments: value }, + }; + } + if (value == null) return { arguments: {} }; + return { + arguments: {}, + extra: { raw_arguments: String(value) }, + }; +} + +function resultContent(value) { + if (value === undefined) return '(missing tool result)'; + if (typeof value === 'string') return value; + try { + const serialized = JSON.stringify(value); + return serialized === undefined ? String(value) : serialized; + } catch { + return String(value); + } +} + +function usageMetrics(usage) { + if (!isObject(usage)) return undefined; + const extra = {}; + // WebBrain records provider-native cost. Some providers report USD, while + // others do not guarantee a currency, so never mislabel it as ATIF cost_usd. + const reportedCost = finiteNumber(usage.cost); + if (reportedCost !== undefined) extra.webbrain_reported_cost = reportedCost; + const metrics = compactObject({ + prompt_tokens: finiteNumber(usage.prompt_tokens), + completion_tokens: finiteNumber(usage.completion_tokens), + cached_tokens: finiteNumber( + usage.cached_tokens ?? usage.prompt_tokens_details?.cached_tokens, + ), + extra: Object.keys(extra).length ? extra : undefined, + }); + return Object.keys(metrics).length ? metrics : undefined; +} + +function eventOrder(left, right) { + const leftSeq = finiteNumber(left?.seq) ?? Number.MAX_SAFE_INTEGER; + const rightSeq = finiteNumber(right?.seq) ?? Number.MAX_SAFE_INTEGER; + if (leftSeq !== rightSeq) return leftSeq - rightSeq; + return (finiteNumber(left?.ts) ?? 0) - (finiteNumber(right?.ts) ?? 0); +} + +function sameToolName(left, right) { + return String(left || '') === String(right || ''); +} + +export function webbrainTraceToAtif(input) { + if (!isObject(input)) throw new TypeError('WebBrain trace export must be an object.'); + if (input.schema !== SOURCE_SCHEMA) { + throw new TypeError(`Expected schema "${SOURCE_SCHEMA}".`); + } + if (!isObject(input.run)) throw new TypeError('WebBrain trace run must be an object.'); + if (!Array.isArray(input.events)) throw new TypeError('WebBrain trace events must be an array.'); + if (typeof input.run.runId !== 'string' || !input.run.runId.trim()) { + throw new TypeError('WebBrain trace run.runId must be a non-empty string.'); + } + + const run = input.run; + const runId = String(run.runId).trim(); + const version = String( + run.webbrainVersion || input.exportedByWebBrainVersion || 'unknown', + ); + const steps = []; + const omittedEventCounts = {}; + let lastAgentMessage = ''; + let pendingAgent = null; + const allocatedToolCallIds = new Set(); + + function appendStep(step) { + const complete = { step_id: steps.length + 1, ...compactObject(step) }; + steps.push(complete); + return complete; + } + + appendStep({ + timestamp: timestamp(run.startedAt), + source: 'user', + message: String(run.userMessage || ''), + }); + + for (const event of [...input.events].sort(eventOrder)) { + if (!isObject(event)) continue; + if (event.runId != null && String(event.runId) !== runId) { + throw new TypeError(`Trace event runId "${event.runId}" does not match run.runId "${runId}".`); + } + const data = isObject(event.data) ? event.data : {}; + const eventSeq = finiteNumber(event.seq); + + if (event.kind === 'llm_response') { + const toolCalls = Array.isArray(data.toolCalls) + ? data.toolCalls.map((call, index) => { + const parsed = parseArguments(call?.args); + const recordedId = String(call?.id || '').trim(); + const fallbackId = `webbrain-${runId}-${eventSeq ?? 'unknown'}-${index + 1}`; + const toolCallId = recordedId && !allocatedToolCallIds.has(recordedId) + ? recordedId + : fallbackId; + allocatedToolCallIds.add(toolCallId); + return compactObject({ + tool_call_id: toolCallId, + function_name: String(call?.name || 'unknown_tool'), + arguments: parsed.arguments, + extra: parsed.extra, + }); + }) + : []; + const message = String(data.content || ''); + const extra = compactObject({ + webbrain_seq: eventSeq, + webbrain_step: finiteNumber(data.step), + phase: data.phase ? String(data.phase) : undefined, + latency_ms: finiteNumber(data.latencyMs), + }); + const step = appendStep({ + timestamp: timestamp(event.ts), + source: 'agent', + model_name: data.model ? String(data.model) : undefined, + message, + tool_calls: toolCalls.length ? toolCalls : undefined, + metrics: usageMetrics(data.usage), + llm_call_count: 1, + extra: Object.keys(extra).length ? extra : undefined, + }); + pendingAgent = { + step, + webbrainStep: finiteNumber(data.step), + usedCallIds: new Set(), + }; + if (message.trim()) lastAgentMessage = message.trim(); + continue; + } + + if (event.kind === 'tool') { + const toolName = String(data.name || 'unknown_tool'); + const pendingCall = pendingAgent?.step.tool_calls?.find((call) => ( + !pendingAgent.usedCallIds.has(call.tool_call_id) + && sameToolName(call.function_name, toolName) + && ( + pendingAgent.webbrainStep === undefined + || finiteNumber(data.step) === undefined + || pendingAgent.webbrainStep === finiteNumber(data.step) + ) + )); + const observationExtra = compactObject({ + latency_ms: finiteNumber(data.latencyMs), + webbrain_seq: eventSeq, + }); + if (pendingCall) { + pendingAgent.usedCallIds.add(pendingCall.tool_call_id); + if (!pendingAgent.step.observation) pendingAgent.step.observation = { results: [] }; + pendingAgent.step.observation.results.push({ + source_call_id: pendingCall.tool_call_id, + content: resultContent(data.result), + ...(Object.keys(observationExtra).length ? { extra: observationExtra } : {}), + }); + continue; + } + + const parsed = parseArguments(data.args); + let toolCallId = `webbrain-${runId}-${eventSeq ?? steps.length + 1}-1`; + let suffix = 2; + while (allocatedToolCallIds.has(toolCallId)) { + toolCallId = `webbrain-${runId}-${eventSeq ?? steps.length + 1}-${suffix}`; + suffix += 1; + } + allocatedToolCallIds.add(toolCallId); + const stepExtra = compactObject({ + webbrain_seq: eventSeq, + webbrain_step: finiteNumber(data.step), + }); + appendStep({ + timestamp: timestamp(event.ts), + source: 'agent', + message: '', + tool_calls: [compactObject({ + tool_call_id: toolCallId, + function_name: toolName, + arguments: parsed.arguments, + extra: parsed.extra, + })], + observation: { + results: [{ + source_call_id: toolCallId, + content: resultContent(data.result), + ...(Object.keys(observationExtra).length ? { extra: observationExtra } : {}), + }], + }, + llm_call_count: 0, + extra: Object.keys(stepExtra).length ? stepExtra : undefined, + }); + pendingAgent = null; + continue; + } + + if (event.kind === 'error') { + const errorExtra = compactObject({ + webbrain_seq: eventSeq, + webbrain_step: finiteNumber(data.step), + phase: data.phase ? String(data.phase) : undefined, + }); + appendStep({ + timestamp: timestamp(event.ts), + source: 'system', + message: 'WebBrain runtime error', + observation: { + results: [{ content: String(data.message || 'Unknown error') }], + }, + extra: Object.keys(errorExtra).length ? errorExtra : undefined, + }); + pendingAgent = null; + continue; + } + + const kind = String(event.kind || 'unknown'); + omittedEventCounts[kind] = (omittedEventCounts[kind] || 0) + 1; + } + + const finalContent = String(run.finalContent || '').trim(); + if (finalContent && finalContent !== lastAgentMessage) { + appendStep({ + timestamp: timestamp(run.endedAt), + source: 'agent', + message: finalContent, + extra: { webbrain_final_content: true }, + }); + } + + const agentExtra = compactObject({ + provider_id: run.providerId ? String(run.providerId) : undefined, + provider_class: run.providerClass ? String(run.providerClass) : undefined, + mode: run.mode ? String(run.mode) : undefined, + }); + const rootExtra = compactObject({ + source_schema: SOURCE_SCHEMA, + source_exported_at: timestamp(input.exportedAt), + source_exported_by_webbrain_version: input.exportedByWebBrainVersion + ? String(input.exportedByWebBrainVersion) + : undefined, + status: run.status ? String(run.status) : undefined, + tab_url: run.tabUrl ? String(run.tabUrl) : undefined, + tab_title: run.tabTitle ? String(run.tabTitle) : undefined, + omitted_event_counts: Object.keys(omittedEventCounts).length + ? omittedEventCounts + : undefined, + }); + const finalMetrics = compactObject({ + total_prompt_tokens: finiteNumber(run.totalInputTokens), + total_completion_tokens: finiteNumber(run.totalOutputTokens), + total_steps: steps.length, + }); + + return { + schema_version: ATIF_SCHEMA_VERSION, + session_id: runId, + trajectory_id: runId, + agent: compactObject({ + name: 'webbrain', + version, + model_name: run.model ? String(run.model) : undefined, + extra: Object.keys(agentExtra).length ? agentExtra : undefined, + }), + steps, + final_metrics: finalMetrics, + extra: rootExtra, + }; +} + +async function main(argv) { + const [inputPath, outputPath] = argv; + if (!inputPath || argv.length > 2) { + throw new Error('Usage: node scripts/trace-to-atif.mjs [trajectory.json|-]'); + } + const raw = await fs.readFile(inputPath, 'utf8'); + const trajectory = webbrainTraceToAtif(JSON.parse(raw)); + const serialized = `${JSON.stringify(trajectory, null, 2)}\n`; + if (outputPath === '-') { + process.stdout.write(serialized); + return; + } + const destination = outputPath || path.join( + path.dirname(inputPath), + `${path.basename(inputPath, path.extname(inputPath))}.atif.json`, + ); + await fs.writeFile(destination, serialized, 'utf8'); + process.stderr.write(`Wrote ${destination}\n`); +} + +const invokedPath = process.argv[1] ? path.resolve(process.argv[1]) : ''; +if (invokedPath === fileURLToPath(import.meta.url)) { + main(process.argv.slice(2)).catch((error) => { + process.stderr.write(`${error?.message || error}\n`); + process.exitCode = 1; + }); +} diff --git a/test/run.js b/test/run.js index 8d284e3c0..6e384b90e 100644 --- a/test/run.js +++ b/test/run.js @@ -8,8 +8,10 @@ */ import { strict as assert } from 'node:assert'; +import { spawnSync } from 'node:child_process'; import fs from 'node:fs'; import { fileURLToPath, pathToFileURL } from 'node:url'; +import os from 'node:os'; import path from 'node:path'; import vm from 'node:vm'; @@ -217,6 +219,9 @@ const { tracesToMarkdown } = await import( const { tracesToMarkdown: tracesToMarkdownFx } = await import( 'file://' + path.join(ROOT, 'src/firefox/src/agent/trace-export.js').replace(/\\/g, '/') ); +const { webbrainTraceToAtif } = await import( + 'file://' + path.join(ROOT, 'scripts/trace-to-atif.mjs').replace(/\\/g, '/') +); const SavedWorkflowsCh = await import( 'file://' + path.join(ROOT, 'src/chrome/src/agent/workflows.js').replace(/\\/g, '/') ); @@ -2716,6 +2721,253 @@ const TRACE_RUNS = [ }, ]; +test('ATIF export: maps a WebBrain run, LLM calls, tools, metrics, and final response', () => { + const input = { + schema: 'webbrain-trace/1', + exportedAt: 1_770_000_100_000, + exportedByWebBrainVersion: '23.4.0', + run: { + runId: 'atif-run-1', + conversationId: 'conversation-7', + startedAt: 1_770_000_000_000, + endedAt: 1_770_000_010_000, + userMessage: 'Find the current title', + model: 'test-model', + providerId: 'local-test', + providerClass: 'openai-compatible', + mode: 'act', + status: 'done', + webbrainVersion: '23.3.1', + totalInputTokens: 12, + totalOutputTokens: 7, + finalContent: 'The title is Example.', + }, + events: [ + { + runId: 'atif-run-1', + seq: 1, + ts: 1_770_000_001_000, + kind: 'llm_response', + data: { + step: 1, + content: '', + model: 'test-model-v2', + latencyMs: 120, + toolCalls: [{ + id: 'call-1', + name: 'read_page', + args: '{"include":"title"}', + }], + usage: { + prompt_tokens: 12, + completion_tokens: 3, + prompt_tokens_details: { cached_tokens: 2 }, + cost: 0.001, + }, + }, + }, + { + runId: 'atif-run-1', + seq: 2, + ts: 1_770_000_002_000, + kind: 'tool', + data: { + step: 1, + name: 'read_page', + args: { include: 'title' }, + result: { title: 'Example' }, + latencyMs: 80, + }, + }, + { + runId: 'atif-run-1', + seq: 3, + ts: 1_770_000_003_000, + kind: 'llm_response', + data: { + step: 2, + content: 'The title is Example.', + usage: { prompt_tokens: 0, completion_tokens: 4 }, + }, + }, + { + runId: 'atif-run-1', + seq: 4, + ts: 1_770_000_004_000, + kind: 'screenshot', + data: { caption: 'viewport', screenshot_base64: 'sensitive-bytes' }, + }, + ], + }; + + const atif = webbrainTraceToAtif(input); + assert.equal(atif.schema_version, 'ATIF-v1.7'); + assert.equal(atif.session_id, 'atif-run-1'); + assert.equal(atif.trajectory_id, 'atif-run-1'); + assert.deepEqual(atif.agent, { + name: 'webbrain', + version: '23.3.1', + model_name: 'test-model', + extra: { + provider_id: 'local-test', + provider_class: 'openai-compatible', + mode: 'act', + }, + }); + assert.equal(atif.steps.length, 3); + assert.deepEqual(atif.steps[0], { + step_id: 1, + timestamp: '2026-02-02T02:40:00.000Z', + source: 'user', + message: 'Find the current title', + }); + assert.equal(atif.steps[1].source, 'agent'); + assert.equal(atif.steps[1].model_name, 'test-model-v2'); + assert.deepEqual(atif.steps[1].tool_calls, [{ + tool_call_id: 'call-1', + function_name: 'read_page', + arguments: { include: 'title' }, + }]); + assert.deepEqual(atif.steps[1].observation.results, [{ + source_call_id: 'call-1', + content: '{"title":"Example"}', + extra: { latency_ms: 80, webbrain_seq: 2 }, + }]); + assert.deepEqual(atif.steps[1].metrics, { + prompt_tokens: 12, + completion_tokens: 3, + cached_tokens: 2, + extra: { webbrain_reported_cost: 0.001 }, + }); + assert.equal(atif.steps[2].message, 'The title is Example.'); + assert.deepEqual(atif.final_metrics, { + total_prompt_tokens: 12, + total_completion_tokens: 7, + total_steps: 3, + }); + assert.deepEqual(atif.extra.omitted_event_counts, { screenshot: 1 }); + assert.ok(!JSON.stringify(atif).includes('sensitive-bytes')); +}); + +test('ATIF export: synthesizes deterministic tool calls and preserves malformed arguments safely', () => { + const input = { + schema: 'webbrain-trace/1', + exportedByWebBrainVersion: '23.4.0', + run: { + runId: 'standalone-tool', + userMessage: '', + model: '', + finalContent: 'Finished.', + }, + events: [ + { + seq: 7, + ts: 1_770_000_007_000, + kind: 'tool', + data: { + step: 1, + name: 'custom_tool', + args: '{not valid json', + result: undefined, + }, + }, + ], + }; + const atif = webbrainTraceToAtif(input); + assert.equal(atif.agent.version, '23.4.0'); + assert.equal(atif.steps[1].llm_call_count, 0); + assert.deepEqual(atif.steps[1].tool_calls, [{ + tool_call_id: 'webbrain-standalone-tool-7-1', + function_name: 'custom_tool', + arguments: {}, + extra: { raw_arguments: '{not valid json' }, + }]); + assert.deepEqual(atif.steps[1].observation.results, [{ + source_call_id: 'webbrain-standalone-tool-7-1', + content: '(missing tool result)', + extra: { webbrain_seq: 7 }, + }]); + assert.equal(atif.steps[2].message, 'Finished.'); +}); + +test('ATIF export: replaces repeated provider tool-call IDs deterministically', () => { + const atif = webbrainTraceToAtif({ + schema: 'webbrain-trace/1', + run: { runId: 'duplicate-calls', userMessage: 'Run both tools' }, + events: [ + { + seq: 1, + kind: 'llm_response', + data: { + step: 1, + toolCalls: [ + { id: 'duplicate', name: 'first', args: '{}' }, + { id: 'duplicate', name: 'second', args: '{}' }, + ], + }, + }, + ], + }); + assert.deepEqual( + atif.steps[1].tool_calls.map((call) => call.tool_call_id), + ['duplicate', 'webbrain-duplicate-calls-1-2'], + ); +}); + +test('ATIF export: rejects unsupported or malformed WebBrain exports', () => { + assert.throws( + () => webbrainTraceToAtif({ schema: 'other/1', run: {}, events: [] }), + /Expected schema "webbrain-trace\/1"/, + ); + assert.throws( + () => webbrainTraceToAtif({ schema: 'webbrain-trace/1', run: {}, events: 'nope' }), + /events must be an array/, + ); + assert.throws( + () => webbrainTraceToAtif({ schema: 'webbrain-trace/1', run: {}, events: [] }), + /run\.runId must be a non-empty string/, + ); + assert.throws( + () => webbrainTraceToAtif({ schema: 'webbrain-trace/1', run: { runId: 7 }, events: [] }), + /run\.runId must be a non-empty string/, + ); + assert.throws( + () => webbrainTraceToAtif({ + schema: 'webbrain-trace/1', + run: { runId: 'expected' }, + events: [{ runId: 'foreign', kind: 'tool', data: {} }], + }), + /event runId "foreign" does not match/, + ); +}); + +test('ATIF export: CLI writes a sibling .atif.json file', () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'webbrain-atif-')); + try { + const sourcePath = path.join(tempDir, 'trace.json'); + fs.writeFileSync(sourcePath, JSON.stringify({ + schema: 'webbrain-trace/1', + exportedByWebBrainVersion: '23.4.0', + run: { runId: 'cli-run', userMessage: 'Hello' }, + events: [], + })); + const result = spawnSync( + process.execPath, + [path.join(ROOT, 'scripts/trace-to-atif.mjs'), sourcePath], + { encoding: 'utf8' }, + ); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stderr, /trace\.atif\.json/); + const output = JSON.parse( + fs.readFileSync(path.join(tempDir, 'trace.atif.json'), 'utf8'), + ); + assert.equal(output.schema_version, 'ATIF-v1.7'); + assert.equal(output.session_id, 'cli-run'); + } finally { + fs.rmSync(tempDir, { recursive: true, force: true }); + } +}); + test('trace export: renders the full tool chain from trace events, in order', () => { const { markdown, turnCount, toolCount } = tracesToMarkdown(TRACE_RUNS); assert.equal(turnCount, 1); From 372e4f96ddffbb668745c29565d525529c6c8bb9 Mon Sep 17 00:00:00 2001 From: alectimison-maker Date: Sun, 26 Jul 2026 16:04:59 +0800 Subject: [PATCH 5/8] feat(traces): add offline OTLP converter --- docs/export-and-workflow-formats.md | 47 +++++ package.json | 1 + scripts/trace-to-otlp.mjs | 316 ++++++++++++++++++++++++++++ test/run.js | 186 ++++++++++++++++ 4 files changed, 550 insertions(+) create mode 100644 scripts/trace-to-otlp.mjs diff --git a/docs/export-and-workflow-formats.md b/docs/export-and-workflow-formats.md index 2ca265135..c79080e91 100644 --- a/docs/export-and-workflow-formats.md +++ b/docs/export-and-workflow-formats.md @@ -59,6 +59,53 @@ Screenshot events can include `screenshot_base64` or `screenshot_dataUrl`. Consumers should check `schema`, tolerate additional fields, and avoid relying on undocumented event internals. +### Convert a trace to OpenTelemetry OTLP JSON + +The repository includes an offline converter for sending an exported +`webbrain-trace/1` run through an OpenTelemetry-compatible observability +pipeline: + +```sh +npm run trace:otlp -- webbrain-trace-example.json \ + --output webbrain-trace-example.otlp.json +``` + +The output is an +[OTLP/HTTP JSON](https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding) +`ExportTraceServiceRequest`. It contains one `invoke_agent WebBrain` root span, +child model-call and `execute_tool` spans, and lightweight lifecycle events. +The mappings follow the current +[OpenTelemetry GenAI semantic conventions](https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-agent-spans.md), +which are still marked development and may change. + +The converter does not upload anything. To send the result to a collector, use +that collector's OTLP/HTTP traces endpoint; for example, for a trusted local +collector: + +```sh +curl -H 'Content-Type: application/json' \ + --data-binary @webbrain-trace-example.otlp.json \ + http://127.0.0.1:4318/v1/traces +``` + +Content capture is off by default. User messages, assistant text, error +messages, tool arguments, and tool results are omitted, while operational +metadata such as model, provider, timings, token counts, tool names, and run +identifiers remains. Screenshots are never embedded in the OTLP output. Add +`--include-content` only when the destination is trusted and the trace has been +reviewed: + +```sh +npm run trace:otlp -- webbrain-trace-example.json \ + --output webbrain-trace-example.otlp.json \ + --include-content +``` + +This is a post-run conversion, not live OpenTelemetry instrumentation. Child +span start times are reconstructed from the recorder's completion timestamp and +reported latency, so they should be used for diagnostics rather than +distributed context propagation. + ## Settings snapshots: `webbrain-config/1` `/export --config` creates a versioned Settings snapshot: diff --git a/package.json b/package.json index f3f5897a8..dae405ba3 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "build:web:landing": "node web/build/build.mjs", "build:web": "npm run build:web:landing && npm run build:blog && npm run build:docs", "build:zip": "node scripts/build-zip.mjs", + "trace:otlp": "node scripts/trace-to-otlp.mjs", "sync:logo": "python3 scripts/sync-logo-assets.py && python3 scripts/gen-store-promos.py && node assets/webstore-explainer-2026/render.mjs", "bump": "node scripts/bump-version.mjs", "release": "node scripts/bump-version.mjs --release" diff --git a/scripts/trace-to-otlp.mjs b/scripts/trace-to-otlp.mjs new file mode 100644 index 000000000..cea36c4fd --- /dev/null +++ b/scripts/trace-to-otlp.mjs @@ -0,0 +1,316 @@ +#!/usr/bin/env node +import { createHash } from 'node:crypto'; +import { readFileSync, writeFileSync } from 'node:fs'; +import { pathToFileURL } from 'node:url'; + +const WEBBRAIN_TRACE_SCHEMA = 'webbrain-trace/1'; +const CONTENT_LIMIT = 20_000; +const SPAN_KIND_INTERNAL = 1; +const SPAN_KIND_CLIENT = 3; +const STATUS_CODE_ERROR = 2; + +function finiteNumber(value, fallback = 0) { + const number = Number(value); + return Number.isFinite(number) ? number : fallback; +} + +function unixNano(milliseconds) { + const micros = BigInt(Math.max(0, Math.round(finiteNumber(milliseconds) * 1000))); + return String(micros * 1000n); +} + +function stableHex(seed, length) { + const value = createHash('sha256').update(String(seed)).digest('hex').slice(0, length); + return /^0+$/.test(value) ? `${'0'.repeat(length - 1)}1` : value; +} + +function safeJson(value) { + try { + const serialized = JSON.stringify(value); + return serialized === undefined ? '' : serialized; + } catch { + try { + return String(value); + } catch { + return '(unserializable)'; + } + } +} + +function boundedContent(value) { + if (value == null) return ''; + const text = typeof value === 'string' ? value : safeJson(value); + if (text.length <= CONTENT_LIMIT) return text; + return `${text.slice(0, CONTENT_LIMIT)}…`; +} + +function anyValue(value) { + if (typeof value === 'boolean') return { boolValue: value }; + if (typeof value === 'number') { + if (!Number.isFinite(value)) return null; + return Number.isInteger(value) + ? { intValue: String(value) } + : { doubleValue: value }; + } + if (value == null || value === '') return null; + return { stringValue: String(value) }; +} + +function attributes(entries) { + return entries.flatMap(([key, value]) => { + const encoded = anyValue(value); + return encoded ? [{ key, value: encoded }] : []; + }); +} + +function eventTime(event, fallback) { + const value = finiteNumber(event?.ts, fallback); + return value > 0 ? value : fallback; +} + +function spanBounds(event, runStart) { + const end = eventTime(event, runStart); + const latency = Math.max(0, finiteNumber(event?.data?.latencyMs)); + return { + startTimeUnixNano: unixNano(Math.max(runStart, end - latency)), + endTimeUnixNano: unixNano(Math.max(runStart, end)), + }; +} + +function failedToolResult(result) { + if (result == null) return true; + return typeof result === 'object' && (result.success === false || Boolean(result.error)); +} + +function contentAttributes(run, includeContent) { + if (!includeContent) return []; + return [ + ['webbrain.user.message', boundedContent(run.userMessage)], + ['webbrain.final.response', boundedContent(run.finalContent)], + ]; +} + +function rootEvents(events, includeContent, runStart) { + return events.flatMap((event) => { + const data = event?.data || {}; + const base = [ + ['webbrain.event.sequence', finiteNumber(event?.seq)], + ['webbrain.step', finiteNumber(data.step)], + ]; + if (event?.kind === 'error') { + return [{ + timeUnixNano: unixNano(eventTime(event, runStart)), + name: 'exception', + attributes: attributes([ + ...base, + ['exception.type', data.phase ? `webbrain.${data.phase}` : 'webbrain.error'], + ...(includeContent ? [['exception.message', boundedContent(data.message)]] : []), + ]), + }]; + } + if (event?.kind === 'streaming' || event?.kind === 'note' || event?.kind === 'screenshot') { + return [{ + timeUnixNano: unixNano(eventTime(event, runStart)), + name: `webbrain.${event.kind}`, + attributes: attributes([ + ...base, + ['webbrain.event.status', data.status], + ['webbrain.event.reason', data.reason], + ...(includeContent && event.kind === 'note' + ? [['webbrain.note', boundedContent(data.note)]] + : []), + ]), + }]; + } + return []; + }); +} + +function inferenceSpan(event, context, includeContent) { + const data = event.data || {}; + const model = String(data.model || context.run.model || 'unknown'); + const usage = data.usage || {}; + return { + traceId: context.traceId, + spanId: stableHex(`${context.run.runId}:llm_response:${event.seq}`, 16), + parentSpanId: context.rootSpanId, + name: `chat ${model}`, + kind: SPAN_KIND_CLIENT, + ...spanBounds(event, context.runStart), + attributes: attributes([ + ['gen_ai.operation.name', 'chat'], + ['gen_ai.provider.name', context.run.providerId], + ['gen_ai.request.model', model], + ['gen_ai.usage.input_tokens', usage.prompt_tokens], + ['gen_ai.usage.output_tokens', usage.completion_tokens], + ['webbrain.event.sequence', finiteNumber(event.seq)], + ['webbrain.step', finiteNumber(data.step)], + ...(includeContent + ? [['webbrain.llm.response.content', boundedContent(data.content)]] + : []), + ]), + }; +} + +function toolSpan(event, context, includeContent) { + const data = event.data || {}; + const name = String(data.name || 'unknown'); + const failed = failedToolResult(data.result); + return { + traceId: context.traceId, + spanId: stableHex(`${context.run.runId}:tool:${event.seq}`, 16), + parentSpanId: context.rootSpanId, + name: `execute_tool ${name}`, + kind: SPAN_KIND_INTERNAL, + ...spanBounds(event, context.runStart), + attributes: attributes([ + ['gen_ai.operation.name', 'execute_tool'], + ['gen_ai.tool.name', name], + ['gen_ai.agent.name', 'WebBrain'], + ...(failed ? [['error.type', 'tool_error']] : []), + ['webbrain.event.sequence', finiteNumber(event.seq)], + ['webbrain.step', finiteNumber(data.step)], + ...(includeContent + ? [ + ['gen_ai.tool.call.arguments', boundedContent(data.args)], + ['gen_ai.tool.call.result', boundedContent(data.result)], + ] + : []), + ]), + ...(failed ? { status: { code: STATUS_CODE_ERROR } } : {}), + }; +} + +export function traceExportToOtlp(input, { includeContent = false } = {}) { + if (!input || input.schema !== WEBBRAIN_TRACE_SCHEMA) { + throw new Error(`Expected a ${WEBBRAIN_TRACE_SCHEMA} export.`); + } + if (!input.run || typeof input.run !== 'object' || Array.isArray(input.run)) { + throw new Error('Trace export must contain a run object.'); + } + if (!Array.isArray(input.events)) { + throw new Error('Trace export must contain an events array.'); + } + + const run = input.run; + const events = [...input.events] + .filter((event) => event && typeof event === 'object') + .sort((a, b) => finiteNumber(a.seq) - finiteNumber(b.seq)); + const eventTimes = events.map((event) => finiteNumber(event.ts)).filter((time) => time > 0); + const fallbackStarts = [...eventTimes, finiteNumber(input.exportedAt)].filter((time) => time > 0); + const initialStart = finiteNumber( + run.startedAt, + fallbackStarts.length ? Math.min(...fallbackStarts) : 0, + ); + const childStarts = events.map((event) => ( + eventTime(event, initialStart) - Math.max(0, finiteNumber(event?.data?.latencyMs)) + )); + const runStart = Math.max(0, Math.min(initialStart, ...childStarts)); + const runEnd = Math.max( + runStart, + finiteNumber(run.endedAt), + runStart + Math.max(0, finiteNumber(run.durationMs)), + ...eventTimes, + ); + const traceId = stableHex(`webbrain:${run.runId || runStart}`, 32); + const rootSpanId = stableHex(`webbrain:${run.runId || runStart}:root`, 16); + const context = { run, runStart, traceId, rootSpanId }; + const hasError = ['error', 'failed', 'loop_stopped'].includes(String(run.status || '').toLowerCase()) + || events.some((event) => event.kind === 'error'); + + const root = { + traceId, + spanId: rootSpanId, + name: 'invoke_agent WebBrain', + kind: SPAN_KIND_INTERNAL, + startTimeUnixNano: unixNano(runStart), + endTimeUnixNano: unixNano(runEnd), + attributes: attributes([ + ['gen_ai.operation.name', 'invoke_agent'], + ['gen_ai.agent.name', 'WebBrain'], + ['gen_ai.agent.version', run.webbrainVersion || input.exportedByWebBrainVersion], + ['gen_ai.provider.name', run.providerId], + ['gen_ai.request.model', run.model], + ['gen_ai.conversation.id', run.conversationId], + ['gen_ai.usage.input_tokens', run.totalInputTokens], + ['gen_ai.usage.output_tokens', run.totalOutputTokens], + ['webbrain.run.id', run.runId], + ['webbrain.run.status', run.status], + ...contentAttributes(run, includeContent), + ]), + events: rootEvents(events, includeContent, runStart), + ...(hasError ? { status: { code: STATUS_CODE_ERROR } } : {}), + }; + const childSpans = events.flatMap((event) => { + if (event.kind === 'llm_response') return [inferenceSpan(event, context, includeContent)]; + if (event.kind === 'tool') return [toolSpan(event, context, includeContent)]; + return []; + }); + + return { + resourceSpans: [{ + resource: { + attributes: attributes([ + ['service.name', 'webbrain'], + ['service.version', run.webbrainVersion || input.exportedByWebBrainVersion], + ]), + }, + scopeSpans: [{ + scope: { + name: 'webbrain.trace-export', + ...(input.exportedByWebBrainVersion + ? { version: String(input.exportedByWebBrainVersion) } + : {}), + }, + spans: [root, ...childSpans], + }], + }], + }; +} + +export function parseTraceToOtlpArgs(argv) { + const parsed = { input: '', output: '', includeContent: false }; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === '--include-content') { + parsed.includeContent = true; + } else if (arg === '--output') { + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error('Missing value for --output.'); + parsed.output = value; + index += 1; + } else if (arg.startsWith('--')) { + throw new Error(`Unknown option: ${arg}`); + } else if (!parsed.input) { + parsed.input = arg; + } else { + throw new Error(`Unexpected argument: ${arg}`); + } + } + if (!parsed.input) throw new Error('Missing input trace JSON path.'); + return parsed; +} + +function runCli() { + try { + const args = parseTraceToOtlpArgs(process.argv.slice(2)); + const input = JSON.parse(readFileSync(args.input, 'utf8')); + const output = `${JSON.stringify( + traceExportToOtlp(input, { includeContent: args.includeContent }), + null, + 2, + )}\n`; + if (args.output) { + writeFileSync(args.output, output); + } else { + process.stdout.write(output); + } + } catch (error) { + console.error(`trace-to-otlp: ${error.message}`); + process.exitCode = 1; + } +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + runCli(); +} diff --git a/test/run.js b/test/run.js index 8d284e3c0..d16db0f21 100644 --- a/test/run.js +++ b/test/run.js @@ -554,6 +554,9 @@ const { } = await import( 'file://' + path.join(ROOT, 'scripts/build-zip.mjs').replace(/\\/g, '/') ); +const { traceExportToOtlp, parseTraceToOtlpArgs } = await import( + 'file://' + path.join(ROOT, 'scripts/trace-to-otlp.mjs').replace(/\\/g, '/') +); // providers/manager.js — pure ESM at module load (chrome.* only inside // methods). We import the class so we can exercise the static categoryFor() @@ -2942,6 +2945,189 @@ test('trace export: renders Ask streaming decisions and aggregate lifecycle metr } }); +const OTLP_TRACE_FIXTURE = { + schema: 'webbrain-trace/1', + exportedAt: 1_784_937_600_000, + exportedByWebBrainVersion: '25.9.7', + run: { + runId: 'run_otlp_fixture', + conversationId: 'conversation_fixture', + startedAt: 1_784_937_600_000, + endedAt: 1_784_937_601_500, + durationMs: 1500, + status: 'loop_stopped', + model: 'test-model', + providerId: 'test-provider', + webbrainVersion: '25.9.6', + userMessage: 'Private user request', + finalContent: 'Private final answer', + }, + events: [ + { + runId: 'run_otlp_fixture', + seq: 1, + ts: 1_784_937_600_500, + kind: 'llm_response', + data: { + step: 1, + model: 'response-model', + latencyMs: 200, + usage: { prompt_tokens: 31, completion_tokens: 12 }, + content: 'Private model response', + }, + }, + { + runId: 'run_otlp_fixture', + seq: 2, + ts: 1_784_937_600_900, + kind: 'tool', + data: { + step: 1, + name: 'fetch_url', + latencyMs: 75, + args: { url: 'https://private.example/account' }, + result: { success: false, error: 'Private tool failure' }, + }, + }, + { + runId: 'run_otlp_fixture', + seq: 3, + ts: 1_784_937_601_000, + kind: 'error', + data: { step: 1, phase: 'loop', message: 'Stopped after repeated failures.' }, + }, + { + runId: 'run_otlp_fixture', + seq: 4, + ts: 1_784_937_601_100, + kind: 'screenshot', + data: { step: 1, screenshot_base64: 'data:image/png;base64,PRIVATE_SCREENSHOT' }, + }, + ], +}; + +function otlpAttributes(items = []) { + return Object.fromEntries(items.map(({ key, value }) => { + const entry = value || {}; + const field = Object.keys(entry)[0]; + return [key, entry[field]]; + })); +} + +test('OTLP trace converter emits valid ID, hierarchy, timing, and GenAI span shapes', () => { + const payload = traceExportToOtlp(OTLP_TRACE_FIXTURE); + assert.equal(payload.resourceSpans.length, 1); + const [{ resource, scopeSpans }] = payload.resourceSpans; + assert.equal(otlpAttributes(resource.attributes)['service.name'], 'webbrain'); + assert.equal(scopeSpans.length, 1); + assert.equal(scopeSpans[0].scope.name, 'webbrain.trace-export'); + + const spans = scopeSpans[0].spans; + assert.equal(spans.length, 3); + const [root, inference, tool] = spans; + assert.match(root.traceId, /^[0-9a-f]{32}$/); + assert.match(root.spanId, /^[0-9a-f]{16}$/); + assert.equal(root.parentSpanId, undefined); + assert.equal(root.name, 'invoke_agent WebBrain'); + assert.equal(root.kind, 1); + assert.equal(root.startTimeUnixNano, '1784937600000000000'); + assert.equal(root.endTimeUnixNano, '1784937601500000000'); + assert.equal(root.status.code, 2); + + for (const child of [inference, tool]) { + assert.equal(child.traceId, root.traceId); + assert.equal(child.parentSpanId, root.spanId); + assert.match(child.spanId, /^[0-9a-f]{16}$/); + } + assert.equal(inference.name, 'chat response-model'); + assert.equal(inference.kind, 3); + assert.equal(inference.startTimeUnixNano, '1784937600300000000'); + assert.equal(inference.endTimeUnixNano, '1784937600500000000'); + assert.deepEqual(otlpAttributes(inference.attributes), { + 'gen_ai.operation.name': 'chat', + 'gen_ai.provider.name': 'test-provider', + 'gen_ai.request.model': 'response-model', + 'gen_ai.usage.input_tokens': '31', + 'gen_ai.usage.output_tokens': '12', + 'webbrain.event.sequence': '1', + 'webbrain.step': '1', + }); + + assert.equal(tool.name, 'execute_tool fetch_url'); + assert.equal(tool.kind, 1); + assert.equal(tool.status.code, 2); + assert.equal(otlpAttributes(tool.attributes)['error.type'], 'tool_error'); + assert.equal(otlpAttributes(tool.attributes)['gen_ai.tool.name'], 'fetch_url'); +}); + +test('OTLP trace converter is deterministic and private-by-default', () => { + const first = traceExportToOtlp(OTLP_TRACE_FIXTURE); + const second = traceExportToOtlp(OTLP_TRACE_FIXTURE); + assert.deepEqual(second, first); + const serialized = JSON.stringify(first); + assert.doesNotMatch(serialized, /Private user request|Private final answer|Private model response/); + assert.doesNotMatch(serialized, /private\.example|Private tool failure/); + assert.doesNotMatch(serialized, /gen_ai\.tool\.call\.(?:arguments|result)/); + + const withContent = traceExportToOtlp(OTLP_TRACE_FIXTURE, { includeContent: true }); + const contentText = JSON.stringify(withContent); + assert.match(contentText, /Private user request/); + assert.match(contentText, /Private final answer/); + assert.match(contentText, /Private model response/); + assert.match(contentText, /gen_ai\.tool\.call\.arguments/); + assert.match(contentText, /gen_ai\.tool\.call\.result/); + assert.doesNotMatch(contentText, /PRIVATE_SCREENSHOT/); +}); + +test('OTLP trace converter rejects other schemas and malformed records', () => { + assert.throws( + () => traceExportToOtlp({ ...OTLP_TRACE_FIXTURE, schema: 'webbrain-trace/2' }), + /webbrain-trace\/1/, + ); + assert.throws( + () => traceExportToOtlp({ ...OTLP_TRACE_FIXTURE, run: null }), + /run object/, + ); + assert.throws( + () => traceExportToOtlp({ ...OTLP_TRACE_FIXTURE, events: {} }), + /events array/, + ); + const legacy = traceExportToOtlp({ + schema: 'webbrain-trace/1', + run: { runId: 'legacy-run' }, + events: [{ + seq: 1, + ts: 1234, + kind: 'tool', + data: { name: 'read_page' }, + }], + }, { includeContent: true }); + const [root, tool] = legacy.resourceSpans[0].scopeSpans[0].spans; + assert.equal(root.startTimeUnixNano, '1234000000'); + assert.equal(tool.startTimeUnixNano, '1234000000'); + assert.doesNotMatch(JSON.stringify(legacy), /stringValue\":\"(?:undefined|null)\"/); +}); + +test('OTLP trace converter CLI parsing keeps content opt-in and output explicit', () => { + assert.deepEqual(parseTraceToOtlpArgs(['trace.json']), { + input: 'trace.json', + output: '', + includeContent: false, + }); + assert.deepEqual(parseTraceToOtlpArgs([ + 'trace.json', + '--output', + 'trace.otlp.json', + '--include-content', + ]), { + input: 'trace.json', + output: 'trace.otlp.json', + includeContent: true, + }); + assert.throws(() => parseTraceToOtlpArgs([]), /input trace JSON/); + assert.throws(() => parseTraceToOtlpArgs(['a.json', '--upload']), /Unknown option/); +}); + test('/export --traces is wired in both side panels and backgrounds', () => { for (const [label, panelRel, bgRel] of [ ['chrome', 'src/chrome/src/ui/sidepanel.js', 'src/chrome/src/background.js'], From 84180d8c50cca0f97ef18f62b08c4e74ac1793ac Mon Sep 17 00:00:00 2001 From: alectimison-maker Date: Sun, 26 Jul 2026 14:40:53 +0800 Subject: [PATCH 6/8] refactor(agent): extract image budget helpers --- TODOs.md | 14 +++--- src/chrome/ARCHITECTURE.md | 1 + src/chrome/src/agent/agent.js | 41 ++-------------- src/chrome/src/agent/image-budget.js | 50 +++++++++++++++++++ src/firefox/ARCHITECTURE.md | 1 + src/firefox/src/agent/agent.js | 37 ++------------ src/firefox/src/agent/image-budget.js | 50 +++++++++++++++++++ test/run.js | 69 +++++++++++---------------- 8 files changed, 144 insertions(+), 119 deletions(-) create mode 100644 src/chrome/src/agent/image-budget.js create mode 100644 src/firefox/src/agent/image-budget.js diff --git a/TODOs.md b/TODOs.md index eba6aa9c3..3ceb666e3 100644 --- a/TODOs.md +++ b/TODOs.md @@ -194,15 +194,15 @@ parity regressions likely. ## 9. Test the real shared logic instead of copied shims -**Status:** Partially resolved. Loop detection now lives in the browser-free -`agent/loop-detector.js` module in both builds. `Agent` inherits that production -class, `test/run.js` imports it directly, and a parity assertion keeps the -Chrome and Firefox copies byte-identical. +**Status:** Partially resolved. Loop detection and image-budget sizing now live +in browser-free modules in both builds. `Agent` consumes the production logic, +`test/run.js` imports it directly, and parity assertions keep the Chrome and +Firefox copies byte-identical. **Concrete next steps:** -1. Move image budget sizing and other remaining pure logic into browser-free - modules and import those modules directly from `agent.js` and `test/run.js`. -2. Add regression tests for the text tool-call parser and context trimming, +1. Move other remaining pure logic into browser-free modules and import those + modules directly from `agent.js` and `test/run.js`. +2. Add broader regression tests for the text tool-call parser and context trimming, since both are high-impact agent reliability code. --- diff --git a/src/chrome/ARCHITECTURE.md b/src/chrome/ARCHITECTURE.md index cf43a56da..6d5c7da5d 100644 --- a/src/chrome/ARCHITECTURE.md +++ b/src/chrome/ARCHITECTURE.md @@ -42,6 +42,7 @@ src/chrome/ │ ├── agent/ │ │ ├── agent.js # Core agent loop + tool dispatch │ │ ├── loop-detector.js # Browser-free loop detection, directly unit-tested +│ │ ├── image-budget.js # Browser-free screenshot sizing, directly unit-tested │ │ ├── mutation-tools.js # This build's state-change + mutating tool sets │ │ ├── tools.js # Tool schemas + system prompts │ │ ├── skills.js # Settings skills + dynamic skill tool manifests diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index 34f39248f..d19a44b29 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -2,6 +2,7 @@ import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, getToolsForMo import { handleDoneJson } from './cloud-output.js'; import { LoopDetector } from './loop-detector.js'; import { parseToolCallsFromText } from './tool-call-parser.js'; +import { IMAGE_BUDGET, estimateImageTokens, fitImageDimensions } from './image-budget.js'; import { BROWSER_MUTATION_TOOLS, STATE_CHANGE_TOOLS as SHARED_STATE_CHANGE_TOOLS } from './mutation-tools.js'; import { isCredentialField, CREDENTIAL_NOTE_STRICT, STRICT_SECRET_SYSTEM_NOTE } from './credential-fields.js'; import { detectProgressAction, formatLedgerRow, formatLedgerSummary, isBlockedLedgerDowngrade, isTerminalLedgerStatus, isValidLedgerStatus, ledgerDoneBlock, ledgerRowKey, normalizeLedgerStatus, progressCounts, selectLedgerRows, unresolvedLedgerRows, upsertLedgerItems } from './progress-ledger.js'; @@ -5747,15 +5748,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // are tuned to Claude's native vision encoder and happen to be // reasonable for most other endpoints too. Override per-capture if you // need sharper (coord_aligned) or looser (full_page) constraints. - static IMAGE_BUDGET = { - pxPerToken: 28, // rough px² per vision token across providers - maxTargetPx: 1568, // no dimension bigger than this - maxTargetTokens: 1568, // total image tokens budget - maxBase64Chars: 1398100, // ~1.4 MB base64, matches Anthropic's cap - initialJpegQuality: 0.75, - minJpegQuality: 0.10, - jpegQualityStep: 0.05, - }; + static IMAGE_BUDGET = IMAGE_BUDGET; /** * Anthropic's token-cost approximation: ceil((w*h) / pxPerToken²). @@ -5763,7 +5756,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d * exact for any specific provider's tokenizer, but better than eyeballing. */ static _estimateImageTokens(w, h, pxPerToken) { - return Math.ceil((w / pxPerToken) * (h / pxPerToken)); + return estimateImageTokens(w, h, pxPerToken); } /** @@ -5773,33 +5766,7 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d * Claude-for-Chrome's `C(w, h, params)` (same algorithm, clearer names). */ static _fitImageDimensions(origW, origH, budget = Agent.IMAGE_BUDGET) { - const { pxPerToken, maxTargetPx, maxTargetTokens } = budget; - // Already fits — no work. - if (origW <= maxTargetPx && origH <= maxTargetPx && - Agent._estimateImageTokens(origW, origH, pxPerToken) <= maxTargetTokens) { - return [origW, origH]; - } - // Search the long side; the other follows from aspect ratio. - if (origH > origW) { - const [h, w] = Agent._fitImageDimensions(origH, origW, budget); - return [w, h]; - } - const aspect = origW / origH; - let hi = origW, lo = 1; - // eslint-disable-next-line no-constant-condition - while (true) { - if (lo + 1 >= hi) { - return [lo, Math.max(Math.round(lo / aspect), 1)]; - } - const mid = Math.floor((lo + hi) / 2); - const midH = Math.max(Math.round(mid / aspect), 1); - if (mid <= maxTargetPx && - Agent._estimateImageTokens(mid, midH, pxPerToken) <= maxTargetTokens) { - lo = mid; - } else { - hi = mid; - } - } + return fitImageDimensions(origW, origH, budget); } /** diff --git a/src/chrome/src/agent/image-budget.js b/src/chrome/src/agent/image-budget.js new file mode 100644 index 000000000..357188f7d --- /dev/null +++ b/src/chrome/src/agent/image-budget.js @@ -0,0 +1,50 @@ +// Browser-free image sizing shared by Agent and the unit suite. This file is +// mirrored in the Firefox tree; keep both copies byte-identical. + +// These defaults match Claude for Chrome. Pixel dimensions and approximate +// vision-token area bound provider cost, while the byte/quality fields are +// consumed by each browser's platform-specific image encoder. +export const IMAGE_BUDGET = { + pxPerToken: 28, + maxTargetPx: 1568, + maxTargetTokens: 1568, + maxBase64Chars: 1398100, + initialJpegQuality: 0.75, + minJpegQuality: 0.10, + jpegQualityStep: 0.05, +}; + +export function estimateImageTokens(w, h, pxPerToken) { + return Math.ceil((w / pxPerToken) * (h / pxPerToken)); +} + +// Return the largest dimensions no greater than the original that fit both +// the per-side and token-area caps while preserving the aspect ratio. +export function fitImageDimensions(origW, origH, budget = IMAGE_BUDGET) { + const { pxPerToken, maxTargetPx, maxTargetTokens } = budget; + if (origW <= maxTargetPx && origH <= maxTargetPx + && estimateImageTokens(origW, origH, pxPerToken) <= maxTargetTokens) { + return [origW, origH]; + } + if (origH > origW) { + const [h, w] = fitImageDimensions(origH, origW, budget); + return [w, h]; + } + const aspect = origW / origH; + let hi = origW; + let lo = 1; + // eslint-disable-next-line no-constant-condition + while (true) { + if (lo + 1 >= hi) { + return [lo, Math.max(Math.round(lo / aspect), 1)]; + } + const mid = Math.floor((lo + hi) / 2); + const midH = Math.max(Math.round(mid / aspect), 1); + if (mid <= maxTargetPx + && estimateImageTokens(mid, midH, pxPerToken) <= maxTargetTokens) { + lo = mid; + } else { + hi = mid; + } + } +} diff --git a/src/firefox/ARCHITECTURE.md b/src/firefox/ARCHITECTURE.md index b40b7300f..26a2902cd 100644 --- a/src/firefox/ARCHITECTURE.md +++ b/src/firefox/ARCHITECTURE.md @@ -55,6 +55,7 @@ src/firefox/ │ ├── agent/ │ │ ├── agent.js # Core agent loop │ │ ├── loop-detector.js # Browser-free loop detection, directly unit-tested +│ │ ├── image-budget.js # Browser-free screenshot sizing, directly unit-tested │ │ ├── mutation-tools.js # This build's state-change + mutating tool sets │ │ ├── tools.js # Tool schemas + system prompts (incl. 4 AX tools) │ │ ├── skills.js # Settings skills + dynamic skill tool manifests diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index 0cee0505c..12dc28819 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -2,6 +2,7 @@ import { AGENT_TOOLS, AGENT_TOOL_NAMES, RESERVED_AGENT_TOOL_NAMES, getToolsForMo import { handleDoneJson } from './cloud-output.js'; import { LoopDetector } from './loop-detector.js'; import { parseToolCallsFromText } from './tool-call-parser.js'; +import { IMAGE_BUDGET, estimateImageTokens, fitImageDimensions } from './image-budget.js'; import { BROWSER_MUTATION_TOOLS, STATE_CHANGE_TOOLS as SHARED_STATE_CHANGE_TOOLS } from './mutation-tools.js'; import { isCredentialField, CREDENTIAL_NOTE_STRICT, STRICT_SECRET_SYSTEM_NOTE } from './credential-fields.js'; import { detectProgressAction, formatLedgerRow, formatLedgerSummary, isBlockedLedgerDowngrade, isTerminalLedgerStatus, isValidLedgerStatus, ledgerDoneBlock, ledgerRowKey, normalizeLedgerStatus, progressCounts, selectLedgerRows, unresolvedLedgerRows, upsertLedgerItems } from './progress-ledger.js'; @@ -4062,44 +4063,14 @@ Rules: no prose intro, no conclusion, no "this screenshot shows...", no layout d // and can use regular / toBlob instead of OffscreenCanvas. // Constants match Anthropic's Claude-for-Chrome defaults and the Chrome // Agent's IMAGE_BUDGET — keep them in sync. - static IMAGE_BUDGET = { - pxPerToken: 28, - maxTargetPx: 1568, - maxTargetTokens: 1568, - maxBase64Chars: 1398100, - initialJpegQuality: 0.75, - minJpegQuality: 0.10, - jpegQualityStep: 0.05, - }; + static IMAGE_BUDGET = IMAGE_BUDGET; static _estimateImageTokens(w, h, pxPerToken) { - return Math.ceil((w / pxPerToken) * (h / pxPerToken)); + return estimateImageTokens(w, h, pxPerToken); } static _fitImageDimensions(origW, origH, budget = Agent.IMAGE_BUDGET) { - const { pxPerToken, maxTargetPx, maxTargetTokens } = budget; - if (origW <= maxTargetPx && origH <= maxTargetPx && - Agent._estimateImageTokens(origW, origH, pxPerToken) <= maxTargetTokens) { - return [origW, origH]; - } - if (origH > origW) { - const [h, w] = Agent._fitImageDimensions(origH, origW, budget); - return [w, h]; - } - const aspect = origW / origH; - let hi = origW, lo = 1; - // eslint-disable-next-line no-constant-condition - while (true) { - if (lo + 1 >= hi) return [lo, Math.max(Math.round(lo / aspect), 1)]; - const mid = Math.floor((lo + hi) / 2); - const midH = Math.max(Math.round(mid / aspect), 1); - if (mid <= maxTargetPx && - Agent._estimateImageTokens(mid, midH, pxPerToken) <= maxTargetTokens) { - lo = mid; - } else { - hi = mid; - } - } + return fitImageDimensions(origW, origH, budget); } /** diff --git a/src/firefox/src/agent/image-budget.js b/src/firefox/src/agent/image-budget.js new file mode 100644 index 000000000..357188f7d --- /dev/null +++ b/src/firefox/src/agent/image-budget.js @@ -0,0 +1,50 @@ +// Browser-free image sizing shared by Agent and the unit suite. This file is +// mirrored in the Firefox tree; keep both copies byte-identical. + +// These defaults match Claude for Chrome. Pixel dimensions and approximate +// vision-token area bound provider cost, while the byte/quality fields are +// consumed by each browser's platform-specific image encoder. +export const IMAGE_BUDGET = { + pxPerToken: 28, + maxTargetPx: 1568, + maxTargetTokens: 1568, + maxBase64Chars: 1398100, + initialJpegQuality: 0.75, + minJpegQuality: 0.10, + jpegQualityStep: 0.05, +}; + +export function estimateImageTokens(w, h, pxPerToken) { + return Math.ceil((w / pxPerToken) * (h / pxPerToken)); +} + +// Return the largest dimensions no greater than the original that fit both +// the per-side and token-area caps while preserving the aspect ratio. +export function fitImageDimensions(origW, origH, budget = IMAGE_BUDGET) { + const { pxPerToken, maxTargetPx, maxTargetTokens } = budget; + if (origW <= maxTargetPx && origH <= maxTargetPx + && estimateImageTokens(origW, origH, pxPerToken) <= maxTargetTokens) { + return [origW, origH]; + } + if (origH > origW) { + const [h, w] = fitImageDimensions(origH, origW, budget); + return [w, h]; + } + const aspect = origW / origH; + let hi = origW; + let lo = 1; + // eslint-disable-next-line no-constant-condition + while (true) { + if (lo + 1 >= hi) { + return [lo, Math.max(Math.round(lo / aspect), 1)]; + } + const mid = Math.floor((lo + hi) / 2); + const midH = Math.max(Math.round(mid / aspect), 1); + if (mid <= maxTargetPx + && estimateImageTokens(mid, midH, pxPerToken) <= maxTargetTokens) { + lo = mid; + } else { + hi = mid; + } + } +} diff --git a/test/run.js b/test/run.js index d462d8cf6..82507853a 100644 --- a/test/run.js +++ b/test/run.js @@ -453,6 +453,16 @@ const ToolCallParserCh = await import( const ToolCallParserFx = await import( 'file://' + path.join(ROOT, 'src/firefox/src/agent/tool-call-parser.js').replace(/\\/g, '/') ); +const ImageBudgetCh = await import( + 'file://' + path.join(ROOT, 'src/chrome/src/agent/image-budget.js').replace(/\\/g, '/') +); +const ImageBudgetFx = await import( + 'file://' + path.join(ROOT, 'src/firefox/src/agent/image-budget.js').replace(/\\/g, '/') +); +const { + estimateImageTokens, + fitImageDimensions, +} = ImageBudgetCh; // The mutating-tool surface differs per build, so it lives outside the // byte-identical loop-detector module. Import the real sets rather than // restating them here — a hand-copied list is exactly the drift this suite @@ -5939,50 +5949,25 @@ test('failed API replay suppresses future bulk replay hints until a success clea } } }); -// -// These mirror the static helpers on Agent exactly — keep them in sync -// with src/chrome/src/agent/agent.js `_estimateImageTokens` and -// `_fitImageDimensions`. We shim rather than import because agent.js -// pulls in chrome.* and cdp-client. -// ──────────────────────────────────────────────────────────────────────── - -const IMAGE_BUDGET_DEFAULT = { - pxPerToken: 28, - maxTargetPx: 1568, - maxTargetTokens: 1568, -}; - -function estimateImageTokens(w, h, pxPerToken) { - return Math.ceil((w / pxPerToken) * (h / pxPerToken)); -} - -function fitImageDimensions(origW, origH, budget = IMAGE_BUDGET_DEFAULT) { - const { pxPerToken, maxTargetPx, maxTargetTokens } = budget; - if (origW <= maxTargetPx && origH <= maxTargetPx && - estimateImageTokens(origW, origH, pxPerToken) <= maxTargetTokens) { - return [origW, origH]; - } - if (origH > origW) { - const [h, w] = fitImageDimensions(origH, origW, budget); - return [w, h]; - } - const aspect = origW / origH; - let hi = origW, lo = 1; - while (true) { - if (lo + 1 >= hi) return [lo, Math.max(Math.round(lo / aspect), 1)]; - const mid = Math.floor((lo + hi) / 2); - const midH = Math.max(Math.round(mid / aspect), 1); - if (mid <= maxTargetPx && - estimateImageTokens(mid, midH, pxPerToken) <= maxTargetTokens) { - lo = mid; - } else { - hi = mid; - } - } -} - console.log('\nimage budget'); +test('image budget helpers are production code shared by both browser agents', () => { + assert.equal( + fs.readFileSync(path.join(ROOT, 'src/chrome/src/agent/image-budget.js'), 'utf8'), + fs.readFileSync(path.join(ROOT, 'src/firefox/src/agent/image-budget.js'), 'utf8'), + 'chrome and firefox image-budget modules must remain byte-identical', + ); + assert.deepEqual(ImageBudgetFx.IMAGE_BUDGET, ImageBudgetCh.IMAGE_BUDGET); + assert.equal( + ImageBudgetFx.estimateImageTokens(1920, 1080, 28), + estimateImageTokens(1920, 1080, 28), + ); + assert.deepEqual( + ImageBudgetFx.fitImageDimensions(3840, 2160), + fitImageDimensions(3840, 2160), + ); +}); + test('small viewport passes through unchanged (fast path)', () => { // 1280×800 at pxPerToken=28 → 1307 tokens < 1568 — no resize. const [w, h] = fitImageDimensions(1280, 800); From 700aef947c3de8b973934bf593c05846f4aaccff Mon Sep 17 00:00:00 2001 From: Emre Sokullu Date: Sun, 26 Jul 2026 21:01:21 +0300 Subject: [PATCH 7/8] refactor(agent): freeze shared image budget defaults Object.freeze the module-level IMAGE_BUDGET so accidental mutation of the shared default throws (modules are strict mode) instead of silently changing every caller. Per-capture overrides already spread into fresh objects. Co-Authored-By: Claude Fable 5 --- src/chrome/src/agent/image-budget.js | 8 +++++--- src/firefox/src/agent/image-budget.js | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/src/chrome/src/agent/image-budget.js b/src/chrome/src/agent/image-budget.js index 357188f7d..d09dcd803 100644 --- a/src/chrome/src/agent/image-budget.js +++ b/src/chrome/src/agent/image-budget.js @@ -3,8 +3,10 @@ // These defaults match Claude for Chrome. Pixel dimensions and approximate // vision-token area bound provider cost, while the byte/quality fields are -// consumed by each browser's platform-specific image encoder. -export const IMAGE_BUDGET = { +// consumed by each browser's platform-specific image encoder. Frozen so an +// accidental mutation of the shared default throws instead of silently +// changing every caller; per-capture overrides spread into a fresh object. +export const IMAGE_BUDGET = Object.freeze({ pxPerToken: 28, maxTargetPx: 1568, maxTargetTokens: 1568, @@ -12,7 +14,7 @@ export const IMAGE_BUDGET = { initialJpegQuality: 0.75, minJpegQuality: 0.10, jpegQualityStep: 0.05, -}; +}); export function estimateImageTokens(w, h, pxPerToken) { return Math.ceil((w / pxPerToken) * (h / pxPerToken)); diff --git a/src/firefox/src/agent/image-budget.js b/src/firefox/src/agent/image-budget.js index 357188f7d..d09dcd803 100644 --- a/src/firefox/src/agent/image-budget.js +++ b/src/firefox/src/agent/image-budget.js @@ -3,8 +3,10 @@ // These defaults match Claude for Chrome. Pixel dimensions and approximate // vision-token area bound provider cost, while the byte/quality fields are -// consumed by each browser's platform-specific image encoder. -export const IMAGE_BUDGET = { +// consumed by each browser's platform-specific image encoder. Frozen so an +// accidental mutation of the shared default throws instead of silently +// changing every caller; per-capture overrides spread into a fresh object. +export const IMAGE_BUDGET = Object.freeze({ pxPerToken: 28, maxTargetPx: 1568, maxTargetTokens: 1568, @@ -12,7 +14,7 @@ export const IMAGE_BUDGET = { initialJpegQuality: 0.75, minJpegQuality: 0.10, jpegQualityStep: 0.05, -}; +}); export function estimateImageTokens(w, h, pxPerToken) { return Math.ceil((w / pxPerToken) * (h / pxPerToken)); From 55876f14d1b2bb5b4beeb31d70acbd4736d0bea1 Mon Sep 17 00:00:00 2001 From: Emre Sokullu Date: Sun, 26 Jul 2026 21:02:17 +0300 Subject: [PATCH 8/8] fix(agent): re-arm duplicate-submit window on acknowledged resubmit An _allowResubmit click previously bypassed the guard without being recorded, so a third rapid duplicate was only blocked while the original entry remained in-window. Acknowledged retries now refresh the entry's timestamp and URL, so each further rapid duplicate needs its own acknowledgement. Helpers stay byte-identical across both trees. Co-Authored-By: Claude Fable 5 --- src/chrome/ARCHITECTURE.md | 2 +- src/chrome/src/agent/submit-click-guard.js | 15 +++++++++--- src/firefox/ARCHITECTURE.md | 6 +++-- src/firefox/src/agent/submit-click-guard.js | 15 +++++++++--- test/run.js | 26 +++++++++++++++++++-- 5 files changed, 53 insertions(+), 11 deletions(-) diff --git a/src/chrome/ARCHITECTURE.md b/src/chrome/ARCHITECTURE.md index 4b9649b6c..a250e5aea 100644 --- a/src/chrome/ARCHITECTURE.md +++ b/src/chrome/ARCHITECTURE.md @@ -419,7 +419,7 @@ System prompt has a new "MODALS & DIALOGS" section that describes the intended f ### Duplicate-submit guard (v3.6.5+) -Before any submit-like text `click`, the agent checks a per-tab+URL 45-second window. Duplicate clicks in that window are blocked unless `_allowResubmit` is set. The browser-free guard is byte-identical in Chrome and Firefox, preventing the "clicked Create three times → three products created" failure mode in both builds. +Before any submit-like text `click`, the agent checks a per-tab+URL 45-second window. Duplicate clicks in that window are blocked unless `_allowResubmit` is set; an acknowledged retry re-arms the window, so a further rapid duplicate needs its own acknowledgement. The browser-free guard is byte-identical in Chrome and Firefox, preventing the "clicked Create three times → three products created" failure mode in both builds. ### API shortcut observer (v18.0.0) diff --git a/src/chrome/src/agent/submit-click-guard.js b/src/chrome/src/agent/submit-click-guard.js index 96b5cab8f..10701fa32 100644 --- a/src/chrome/src/agent/submit-click-guard.js +++ b/src/chrome/src/agent/submit-click-guard.js @@ -8,8 +8,9 @@ const SUBMIT_LIKE_CLICK_RE = /^(create|save|submit|add|post|publish|send|confirm /** * Record the first submit-like text click and reject a rapid duplicate on the * same tab and URL. Returns the tool result to emit when blocked, otherwise - * null. The URL callback is lazy so ordinary clicks and explicit retries do - * not perform a tab lookup. + * null. The URL callback is lazy so ordinary clicks do not perform a tab + * lookup. An `_allowResubmit` retry is allowed through but re-recorded, so a + * further rapid duplicate needs its own acknowledgement. */ export async function guardRecentSubmitClick( recentSubmitClicks, @@ -18,7 +19,7 @@ export async function guardRecentSubmitClick( getCurrentUrl, now = Date.now, ) { - if (!args?.text || args._allowResubmit) return null; + if (!args?.text) return null; const rawText = String(args.text).trim(); if (!SUBMIT_LIKE_CLICK_RE.test(rawText)) return null; @@ -34,6 +35,14 @@ export async function guardRecentSubmitClick( const fresh = entries.filter(entry => timestamp - entry.ts < SUBMIT_CLICK_WINDOW_MS); const match = fresh.find(entry => entry.key === key); + if (match && args._allowResubmit) { + match.ts = timestamp; + match.url = currentUrl; + match.text = rawText; + recentSubmitClicks.set(tabId, fresh); + return null; + } + if (match) { return { success: false, diff --git a/src/firefox/ARCHITECTURE.md b/src/firefox/ARCHITECTURE.md index 0a113f66d..bf3ccf02b 100644 --- a/src/firefox/ARCHITECTURE.md +++ b/src/firefox/ARCHITECTURE.md @@ -193,8 +193,10 @@ System prompt has a new "MODALS & DIALOGS" section describing the intended flow Submit-like text clicks use the same browser-free guard as Chrome. The first click is recorded by tab, normalized label, and current URL; another matching click within 45 seconds is blocked unless `_allowResubmit` explicitly -acknowledges the retry. Navigation, expired entries, ordinary labels, and -validation-rejected submits remain eligible for a fresh click. +acknowledges the retry. An acknowledged retry re-arms the window, so a further +rapid duplicate needs its own acknowledgement. Navigation, expired entries, +ordinary labels, and validation-rejected submits remain eligible for a fresh +click. --- diff --git a/src/firefox/src/agent/submit-click-guard.js b/src/firefox/src/agent/submit-click-guard.js index 96b5cab8f..10701fa32 100644 --- a/src/firefox/src/agent/submit-click-guard.js +++ b/src/firefox/src/agent/submit-click-guard.js @@ -8,8 +8,9 @@ const SUBMIT_LIKE_CLICK_RE = /^(create|save|submit|add|post|publish|send|confirm /** * Record the first submit-like text click and reject a rapid duplicate on the * same tab and URL. Returns the tool result to emit when blocked, otherwise - * null. The URL callback is lazy so ordinary clicks and explicit retries do - * not perform a tab lookup. + * null. The URL callback is lazy so ordinary clicks do not perform a tab + * lookup. An `_allowResubmit` retry is allowed through but re-recorded, so a + * further rapid duplicate needs its own acknowledgement. */ export async function guardRecentSubmitClick( recentSubmitClicks, @@ -18,7 +19,7 @@ export async function guardRecentSubmitClick( getCurrentUrl, now = Date.now, ) { - if (!args?.text || args._allowResubmit) return null; + if (!args?.text) return null; const rawText = String(args.text).trim(); if (!SUBMIT_LIKE_CLICK_RE.test(rawText)) return null; @@ -34,6 +35,14 @@ export async function guardRecentSubmitClick( const fresh = entries.filter(entry => timestamp - entry.ts < SUBMIT_CLICK_WINDOW_MS); const match = fresh.find(entry => entry.key === key); + if (match && args._allowResubmit) { + match.ts = timestamp; + match.url = currentUrl; + match.text = rawText; + recentSubmitClicks.set(tabId, fresh); + return null; + } + if (match) { return { success: false, diff --git a/test/run.js b/test/run.js index 28e5fcdab..fb107ad03 100644 --- a/test/run.js +++ b/test/run.js @@ -34089,7 +34089,7 @@ test('duplicate submit-click guard is production code with Chrome/Firefox parity ); assert.equal(recentClicks.get(tabId).length, 1, 'expired submit-click entries were not pruned'); - const readsBeforeIgnoredCalls = urlReads; + now += 30_000; assert.equal( await guardModule.guardRecentSubmitClick( recentClicks, @@ -34101,6 +34101,28 @@ test('duplicate submit-click guard is production code with Chrome/Firefox parity null, 'explicit resubmit acknowledgement should bypass the guard', ); + assert.equal( + recentClicks.get(tabId)[0].ts, + now, + 'an acknowledged resubmit should re-record the entry', + ); + + now += 30_000; + const thirdRapidClick = await guardModule.guardRecentSubmitClick( + recentClicks, + tabId, + { text: 'Save changes' }, + getCurrentUrl, + () => now, + ); + assert.equal( + thirdRapidClick.blockedDuplicateSubmit, + true, + 'a rapid duplicate after an acknowledged resubmit should be blocked by the re-armed window', + ); + assert.equal(thirdRapidClick.secondsSincePrevious, 30); + + const readsBeforeIgnoredCalls = urlReads; assert.equal( await guardModule.guardRecentSubmitClick( recentClicks, @@ -34112,7 +34134,7 @@ test('duplicate submit-click guard is production code with Chrome/Firefox parity null, 'non-submit-like labels should bypass the guard', ); - assert.equal(urlReads, readsBeforeIgnoredCalls, 'bypassed clicks should not query the tab URL'); + assert.equal(urlReads, readsBeforeIgnoredCalls, 'non-submit-like clicks should not query the tab URL'); const failedLookupClicks = new Map(); const failingUrlLookup = async () => {