diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 002322fc86..8cbd2bebbd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -98,6 +98,7 @@ jobs: - run: pnpm build - run: git diff --exit-code -- packages/protocol/stagehand.v4.json - run: pnpm check + - run: pnpm test:unit - run: uv lock --check working-directory: packages/sdk-python - run: uv run --locked python scripts/generate.py --check diff --git a/packages/docs/v4/reference/stagehand.mdx b/packages/docs/v4/reference/stagehand.mdx index 6c59691c70..63d1738185 100644 --- a/packages/docs/v4/reference/stagehand.mdx +++ b/packages/docs/v4/reference/stagehand.mdx @@ -41,6 +41,42 @@ const stagehand = await Stagehand.create({ browser }); A browser handle from `browserbase.launch()`, `browserbase.connect()`, `localBrowser.launch()`, or `localBrowser.connect()`. Each handle can back only one Stagehand instance. + + Browserbase API key used by managed services such as Model Gateway and server-side caching. + + + + Stagehand API origin override for managed services. Use the service origin without `/v1`. + + + + Default model configuration or client-provided generation callback. Browserbase selects a model automatically for Gateway sessions when omitted. + + + + OpenTelemetry trace export configuration. + + + + Additional system instructions included in model calls. + + + + Whether Stagehand re-infers and retries a cached action when its recorded selector no longer resolves. + + + + Maximum time in milliseconds to wait for the DOM to settle before an operation. + + + + Instance-level server-side cache setting for `act()`, `observe()`, and `extract()`. + + + + Client-side log level, output format, and optional log callback. + + An initialized Stagehand instance. @@ -854,6 +890,50 @@ stagehand = await Stagehand.create(browser=browser) A browser handle from `browserbase.launch()`, `browserbase.connect()`, `local_browser.launch()`, or `local_browser.connect()`. Each handle can back only one Stagehand instance. + + Browserbase API key used by managed services such as Model Gateway and server-side caching. + + + + Stagehand API origin override for managed services. Use the service origin without `/v1`. + + + + Default provider-prefixed model name or client-provided generation callback. Browserbase selects a model automatically for Gateway sessions when omitted. + + + + Provider API key for the model named by `model`. It cannot be used with a generation callback. + + + + Custom provider headers for the model named by `model`. They cannot be used with a generation callback. + + + + OpenTelemetry trace export configuration. + + + + Additional system instructions included in model calls. + + + + Whether Stagehand re-infers and retries a cached action when its recorded selector no longer resolves. + + + + Maximum time in milliseconds to wait for the DOM to settle before an operation. + + + + Instance-level server-side cache setting for `act()`, `observe()`, and `extract()`. + + + + Client-side log level, output format, and optional log callback. + + An initialized Stagehand instance. @@ -1631,7 +1711,45 @@ func Create(ctx context.Context, options CreateOptions) (*Stagehand, error) A browser handle from `LaunchBrowserbase()`, `ConnectBrowserbase()`, `LaunchLocalBrowser()`, or `ConnectLocalBrowser()`. Each handle can back only one Stagehand instance. -The remaining `CreateOptions` fields (`APIKey`, `APIURL`, `Cache`, `DOMSettleTimeoutMs`, `Model`, `Generate`, `Logging`, `SelfHeal`, `SystemPrompt`, and `Telemetry`) are optional and covered in the configuration guides. + + Browserbase API key used by managed services such as Model Gateway and server-side caching. + + + + Stagehand API origin override for managed services. Use the service origin without `/v1`. + + + + Default model configuration. Browserbase selects a model automatically for Gateway sessions when omitted. + + + + Client-provided generation callback used instead of `options.Model`. + + + + OpenTelemetry trace export configuration. + + + + Additional system instructions included in model calls. + + + + Whether Stagehand re-infers and retries a cached action when its recorded selector no longer resolves. + + + + Maximum time in milliseconds to wait for the DOM to settle before an operation. + + + + Instance-level server-side cache setting for `Act()`, `Observe()`, and `Extract()`. + + + + Client-side log level, output format, and optional log callback. + An initialized Stagehand instance. diff --git a/rules/ast-grep/sdk-client-schema-parity.test.ts b/rules/ast-grep/sdk-client-schema-parity.test.ts new file mode 100644 index 0000000000..da604a10db --- /dev/null +++ b/rules/ast-grep/sdk-client-schema-parity.test.ts @@ -0,0 +1,426 @@ +import { readFile } from "node:fs/promises"; +import go from "@ast-grep/lang-go"; +import python from "@ast-grep/lang-python"; +import { parse, registerDynamicLanguage, type SgNode } from "@ast-grep/napi"; +import { describe, expect, it } from "vitest"; +import * as ClientSchemas from "../../packages/sdk-ts/src/clientSchemas.js"; + +registerDynamicLanguage({ go, python }); + +type ObjectSchema = { + shape: Record; +}; + +type Concept = { + go: () => Promise; + name: string; + python: () => Promise; + typescript: ObjectSchema; +}; + +const pythonSource = new URL("../../packages/sdk-python/src/stagehand/", import.meta.url); +const goSource = new URL("../../packages/sdk-go/", import.meta.url); +const docsSource = new URL("../../packages/docs/v4/", import.meta.url); +// These legacy Chrome extension ID overrides are not user-actionable and are pending deprecation. +const intentionallyUndocumentedBrowserFields = new Set([ + "LocalBrowserConnectOptions.extension_id", + "BrowserbaseConnectOptions.extension_id", +]); + +const concepts: readonly Concept[] = [ + { + name: "LocalBrowserLaunchOptions", + typescript: ClientSchemas.LocalBrowserLaunchOptionsSchema, + python: () => pythonClassFields("client_types.py", "LocalBrowserLaunchOptions"), + go: () => goStructFields("browser_factories.go", "LocalBrowserLaunchOptions"), + }, + { + name: "LocalBrowserConnectOptions", + typescript: ClientSchemas.LocalBrowserConnectOptionsSchema, + python: () => pythonClassFields("client_types.py", "LocalBrowserConnectOptions"), + go: () => goStructFields("browser_factories.go", "LocalBrowserConnectOptions"), + }, + { + name: "BrowserbaseConnectOptions", + typescript: ClientSchemas.BrowserbaseConnectOptionsSchema, + python: () => pythonClassFields("client_types.py", "BrowserbaseConnectOptions"), + go: () => goStructFields("browser_factories.go", "BrowserbaseConnectOptions"), + }, + { + name: "StagehandClientLoggingConfig", + typescript: ClientSchemas.StagehandClientLoggingConfigSchema, + python: () => pythonClassFields("client_types.py", "StagehandClientLoggingConfig"), + go: () => goStructFields("client_options.go", "StagehandClientLoggingConfig"), + }, + { + name: "StagehandClientActOptions", + typescript: ClientSchemas.StagehandClientActOptionsSchema, + python: () => pythonMethodParameters("stagehand.py", "Stagehand", "act", ["instruction"]), + go: () => goStructFields("client_options.go", "StagehandClientActOptions"), + }, + { + name: "StagehandClientObserveOptions", + typescript: ClientSchemas.StagehandClientObserveOptionsSchema, + python: () => pythonMethodParameters("stagehand.py", "Stagehand", "observe", ["instruction"]), + go: () => goStructFields("client_options.go", "StagehandClientObserveOptions"), + }, + { + name: "StagehandClientExtractOptions", + typescript: ClientSchemas.StagehandClientExtractOptionsSchema, + python: () => + pythonMethodParameters("stagehand.py", "Stagehand", "extract", ["instruction", "schema"]), + go: () => goStructFields("client_options.go", "StagehandClientExtractOptions"), + }, +] as const; + +describe("SDK-owned schemas remain one cross-language contract", () => { + it("keeps every comparable SDK-only field in TypeScript, Python, and Go", async () => { + const differences: string[] = []; + + for (const concept of concepts) { + const expected = schemaFields(concept.typescript); + const [pythonFields, goFields] = await Promise.all([concept.python(), concept.go()]); + if (!arraysEqual(pythonFields, expected)) { + differences.push( + `${concept.name} Python: expected [${expected.join(", ")}], received [${pythonFields.join(", ")}]`, + ); + } + if (!arraysEqual(goFields, expected)) { + differences.push( + `${concept.name} Go: expected [${expected.join(", ")}], received [${goFields.join(", ")}]`, + ); + } + } + + expect( + differences, + "SDK-only field names are derived dynamically; only concept/type names are paired explicitly", + ).toEqual([]); + }); + + it("keeps Python public input declarations and runtime models in sync", async () => { + const pairs = [ + "LocalBrowserLaunchOptions", + "LocalBrowserConnectOptions", + "BrowserbaseConnectOptions", + "StagehandClientLoggingConfig", + "StagehandClientCreateConfig", + ]; + const differences: string[] = []; + + for (const name of pairs) { + const [input, runtime] = await Promise.all([ + pythonClassFields("client_types.py", name), + pythonClassFields("client_models.py", name), + ]); + if (!arraysEqual(input, runtime)) { + differences.push( + `${name}: public input [${input.join(", ")}], runtime model [${runtime.join(", ")}]`, + ); + } + } + + expect( + differences, + "Python TypedDicts and Pydantic models must expose identical fields", + ).toEqual([]); + }); + + it("keeps Stagehand creation fields aligned while naming intentional language adapters", async () => { + const expectedConfig = schemaFields(ClientSchemas.StagehandClientCreateConfigSchema); + const expectedCreate = schemaFields(ClientSchemas.StagehandCreateOptionsSchema); + const [pythonConfig, pythonCreate, goCreate] = await Promise.all([ + pythonClassFields("client_models.py", "StagehandClientCreateConfig"), + pythonMethodParameters("stagehand.py", "Stagehand", "create", [ + "model_api_key", + "model_headers", + ]), + goStructFields("client_options.go", "CreateOptions"), + ]); + + expect(pythonConfig, "Python create config must match the canonical client schema").toEqual( + expectedConfig, + ); + expect(pythonCreate, "Python create parameters must match StagehandCreateOptions").toEqual( + expectedCreate, + ); + expect( + goCreate.filter((field) => field !== "generate"), + "Go splits the client-LLM callback into Generate; every other CreateOptions field must match", + ).toEqual(expectedCreate); + expect(goCreate, "Go must retain its explicit client-LLM callback adapter").toContain( + "generate", + ); + }); + + it("classifies every exported SDK-owned object schema", () => { + const compared = new Set([ + ...concepts.map(({ name }) => `${name}Schema`), + "StagehandClientCreateConfigSchema", + "StagehandCreateOptionsSchema", + ]); + const intentionallySpecialized = new Set([ + // Browserbase owns this open pass-through surface. + "BrowserbaseLaunchOptionsSchema", + // These validate Browserbase SDK responses rather than cross-language caller input. + "BrowserbaseSessionConnectionSchema", + "BrowserbaseSessionCreateResultSchema", + "BrowserbaseSessionRetrieveResultSchema", + // Runtime callbacks and handles are necessarily language-specific. + "ClientLLMSchema", + // These are partial views of already-generated protocol schemas. + "WebMCPInvokeOptionsSchema", + "WebMCPResultOptionsSchema", + "WebMCPToolsOptionsSchema", + ]); + const objectSchemas = Object.entries(ClientSchemas) + .filter( + ([name, value]) => + name.endsWith("Schema") && + typeof value === "object" && + value !== null && + "shape" in value, + ) + .map(([name]) => name) + .sort(); + + expect( + objectSchemas, + "A new SDK-owned Zod object must join cross-language parity or be explicitly classified", + ).toStrictEqual([...compared, ...intentionallySpecialized].sort()); + }); + + it("documents every Stagehand creation field structurally in each language tab", async () => { + const source = await readFile(new URL("reference/stagehand.mdx", docsSource), "utf8"); + const pythonFields = await pythonMethodParameters("stagehand.py", "Stagehand", "create"); + const expected = { + TypeScript: Object.keys(ClientSchemas.StagehandCreateOptionsSchema.shape).sort(), + Python: pythonFields, + Go: (await goStructFieldSpellings("client_options.go", "CreateOptions")) + .map((field) => `options.${field}`) + .sort(), + } as const; + const differences: string[] = []; + + for (const [language, fields] of Object.entries(expected)) { + const section = languageSection(source, language); + const create = headingSection(section, language === "Go" ? "Create" : "create"); + const documented = [...create.matchAll(/ match[1] as string) + .sort(); + if (!arraysEqual(documented, fields)) { + differences.push( + `${language}: expected [${fields.join(", ")}], received [${documented.join(", ")}]`, + ); + } + } + + expect( + differences, + "Stagehand.create reference fields must be exhaustive even when guides provide the longer explanations", + ).toEqual([]); + }); + + it("mentions every SDK-owned browser and logging field in the configuration docs", async () => { + const [browserDocs, loggingDocs] = await Promise.all([ + readFile(new URL("configuration/browser.mdx", docsSource), "utf8"), + readFile(new URL("configuration/logging.mdx", docsSource), "utf8"), + ]); + const missing: string[] = []; + const browserConcepts = concepts.filter(({ name }) => + [ + "LocalBrowserLaunchOptions", + "LocalBrowserConnectOptions", + "BrowserbaseConnectOptions", + ].includes(name), + ); + + for (const concept of browserConcepts) { + const typescriptFields = Object.keys(concept.typescript.shape); + const pythonFields = await concept.python(); + const goFile = + concept.name === "StagehandClientLoggingConfig" + ? "client_options.go" + : "browser_factories.go"; + const goFields = await goStructFieldSpellings(goFile, concept.name); + for (const field of typescriptFields) { + if ( + !isIntentionallyUndocumentedBrowserField(concept.name, field) && + !browserDocs.includes(field) + ) { + missing.push(`browser TypeScript ${concept.name}.${field}`); + } + } + for (const field of pythonFields) { + if ( + !isIntentionallyUndocumentedBrowserField(concept.name, field) && + !browserDocs.includes(field) + ) { + missing.push(`browser Python ${concept.name}.${field}`); + } + } + for (const field of goFields) { + if ( + !isIntentionallyUndocumentedBrowserField(concept.name, field) && + !browserDocs.includes(field) + ) { + missing.push(`browser Go ${concept.name}.${field}`); + } + } + } + for (const field of Object.keys(ClientSchemas.StagehandClientLoggingConfigSchema.shape)) { + if (!loggingDocs.includes(field)) missing.push(`logging TypeScript ${field}`); + } + for (const field of await pythonClassFields( + "client_types.py", + "StagehandClientLoggingConfig", + )) { + if (!loggingDocs.includes(field)) missing.push(`logging Python ${field}`); + } + for (const field of await goStructFieldSpellings( + "client_options.go", + "StagehandClientLoggingConfig", + )) { + if (!loggingDocs.includes(field)) missing.push(`logging Go ${field}`); + } + + expect( + missing, + "Configuration docs must mention every SDK-owned field in each language's public spelling", + ).toEqual([]); + }); +}); + +function schemaFields(schema: ObjectSchema): string[] { + return Object.keys(schema.shape).map(snakeCase).sort(); +} + +function isIntentionallyUndocumentedBrowserField(concept: string, field: string): boolean { + return intentionallyUndocumentedBrowserFields.has(`${concept}.${snakeCase(field)}`); +} + +async function pythonClassFields(file: string, className: string): Promise { + const root = parse("python", await readFile(new URL(file, pythonSource), "utf8")).root(); + const classNode = root + .findAll({ rule: { kind: "class_definition" } }) + .find((candidate) => candidate.field("name")?.text() === className); + if (!classNode) throw new Error(`${className} was not found in ${file}`); + const body = classNode.field("body"); + if (!body) return []; + return [ + ...new Set( + body + .findAll({ rule: { kind: "assignment" } }) + .filter( + (assignment) => + assignment + .ancestors() + .find((ancestor) => ancestor.kind() === "class_definition") + ?.field("name") + ?.text() === className, + ) + .flatMap((assignment) => { + const left = assignment.field("left") ?? namedChildren(assignment)[0]; + return left?.kind() === "identifier" && left.text() !== "model_config" + ? [snakeCase(left.text())] + : []; + }), + ), + ].sort(); +} + +async function pythonMethodParameters( + file: string, + className: string, + methodName: string, + excluded: readonly string[] = [], +): Promise { + const root = parse("python", await readFile(new URL(file, pythonSource), "utf8")).root(); + const method = root.findAll({ rule: { kind: "function_definition" } }).find( + (candidate) => + candidate.field("name")?.text() === methodName && + candidate + .ancestors() + .find((ancestor) => ancestor.kind() === "class_definition") + ?.field("name") + ?.text() === className && + !candidate + .ancestors() + .some( + (ancestor) => + ancestor.kind() === "decorated_definition" && ancestor.text().startsWith("@overload"), + ), + ); + if (!method) throw new Error(`${className}.${methodName} was not found in ${file}`); + const parameters = method.field("parameters"); + if (!parameters) return []; + const ignored = new Set(["self", "cls", ...excluded].map(snakeCase)); + return [ + ...new Set( + namedChildren(parameters) + .flatMap((parameter) => { + const name = pythonParameterName(parameter); + return name ? [snakeCase(name)] : []; + }) + .filter((name) => !ignored.has(name)), + ), + ].sort(); +} + +function pythonParameterName(parameter: SgNode): string | undefined { + if (parameter.kind() === "identifier") return parameter.text(); + const name = parameter.field("name") ?? parameter.field("pattern"); + if (name) return name.find({ rule: { kind: "identifier" } })?.text() ?? name.text(); + return parameter.find({ rule: { kind: "identifier" } })?.text(); +} + +async function goStructFields(file: string, structName: string): Promise { + return (await goStructFieldSpellings(file, structName)).map(snakeCase).sort(); +} + +async function goStructFieldSpellings(file: string, structName: string): Promise { + const source = await readFile(new URL(file, goSource), "utf8"); + const body = source.match( + new RegExp(`^type ${structName} struct \\{\\n([\\s\\S]*?)^\\}`, "mu"), + )?.[1]; + if (body === undefined) throw new Error(`${structName} was not found in ${file}`); + return body + .split("\n") + .flatMap((line) => line.match(/^\s*([A-Z][A-Za-z0-9_]*)\s+/u)?.[1] ?? []) + .sort(); +} + +function languageSection(source: string, language: string): string { + const start = source.indexOf(``); + if (start < 0) throw new Error(`Missing ${language} tab`); + const next = source.indexOf(" value === right[index]); +} + +function snakeCase(value: string): string { + let normalized = value; + for (const acronym of ["HTTPS", "HTTP", "CDP", "API", "URL", "LLM", "MCP", "MIME", "ID"]) { + normalized = normalized.replaceAll(acronym, `_${acronym.toLowerCase()}_`); + } + return normalized + .replace(/([a-z\d])([A-Z])/gu, "$1_$2") + .replace(/([A-Z]+)([A-Z][a-z])/gu, "$1_$2") + .replace(/[-_]+/gu, "_") + .replace(/^_|_$/gu, "") + .toLowerCase(); +} + +function namedChildren(node: SgNode): SgNode[] { + return node.children().filter((child) => child.isNamed()); +} diff --git a/rules/ast-grep/sdk-field-pipeline.test.ts b/rules/ast-grep/sdk-field-pipeline.test.ts new file mode 100644 index 0000000000..52a78c9410 --- /dev/null +++ b/rules/ast-grep/sdk-field-pipeline.test.ts @@ -0,0 +1,558 @@ +import { readdir, readFile } from "node:fs/promises"; +import { basename } from "node:path"; +import go from "@ast-grep/lang-go"; +import python from "@ast-grep/lang-python"; +import { parse, registerDynamicLanguage, type SgNode } from "@ast-grep/napi"; +import { describe, expect, it } from "vitest"; + +registerDynamicLanguage({ go, python }); + +type Language = "go" | "python" | "typescript"; + +type JsonSchema = { + $ref?: string; + allOf?: JsonSchema[]; + anyOf?: JsonSchema[]; + const?: unknown; + oneOf?: JsonSchema[]; + properties?: Record; + type?: string | string[]; +}; + +type ProtocolMethod = { + properties: { + params: JsonSchema; + result: JsonSchema; + }; +}; + +type ProtocolDocument = JsonSchema & { + $defs: Record; + properties: { + methods: { properties: Record }; + }; +}; + +type RpcCall = { + call: SgNode; + file: string; + language: Language; + module: SgNode; + params: SgNode; + result?: SgNode; + scope: SgNode; + wireMethod: string; +}; + +const sources = { + typescript: new URL("../../packages/sdk-ts/src/", import.meta.url), + python: new URL("../../packages/sdk-python/src/stagehand/", import.meta.url), + go: new URL("../../packages/sdk-go/", import.meta.url), +} as const; +const protocolUrl = new URL("../../packages/protocol/stagehand.v4.json", import.meta.url); +const registryUrl = new URL("../../packages/protocol/schema-registry.ts", import.meta.url); +const referenceUrl = new URL("../../packages/docs/v4/reference/", import.meta.url); +const intentionallyUnusedResultFields = new Set(["page.screenshot.type"]); + +describe("Every public SDK field participates in the protocol pipeline", () => { + it("constructs every declared request field in every SDK", async () => { + const [protocol, calls, helperBodies] = await Promise.all([ + protocolDocument(), + publicRpcCalls(), + callableBodies(), + ]); + const missing: string[] = []; + + for (const call of calls) { + const method = protocol.properties.methods.properties[call.wireMethod]; + if (!method) continue; + const params = resolveSchema(protocol, method.properties.params); + const topLevelFields = Object.keys(schemaProperties(protocol, params)); + if (topLevelFields.length === 0 || isWholeParamsReference(call.params)) continue; + + const scopeText = call.scope.text(); + const tokens = semanticTokens( + `${scopeText}\n${relatedHelperBodies(call.scope, call.language, helperBodies)}`, + ); + const descriptorFields = Object.keys( + schemaProperties(protocol, protocol.$defs.LocatorDescriptor ?? {}), + ); + + for (const field of topLevelFields) { + const normalized = snakeCase(field); + const coveredByDescriptor = + /descriptor/u.test(scopeText) && + descriptorFields.some((descriptorField) => snakeCase(descriptorField) === normalized); + if (!tokens.has(normalized) && !coveredByDescriptor) { + missing.push(`${call.language} ${call.wireMethod}: request field ${field}`); + } + } + + for (const [field, fieldSchema] of Object.entries(schemaProperties(protocol, params))) { + const nestedFields = nestedFieldNames(protocol, fieldSchema); + if (nestedFields.length === 0) continue; + if (directlyForwardsParameter(call.params, field, scopeText)) continue; + + const extendedText = `${scopeText}\n${relatedHelperBodies( + call.scope, + call.language, + helperBodies, + )}`; + // A rest spread forwards newly added fields automatically. Explicitly transformed + // fields remain visible beside the spread. + if (/\.\.\.[A-Za-z_$][A-Za-z0-9_$]*\b/u.test(extendedText)) continue; + const extendedTokens = semanticTokens(extendedText); + for (const nestedField of nestedFields) { + if (!extendedTokens.has(snakeCase(nestedField))) { + missing.push( + `${call.language} ${call.wireMethod}: nested request field ${field}.${nestedField}`, + ); + } + } + } + } + + expect( + missing.sort(), + "Every protocol request field must be visibly constructed, forwarded wholesale, or handled by a complete adapter", + ).toEqual([]); + }); + + it("consumes every declared result field in every SDK", async () => { + const [protocol, calls] = await Promise.all([protocolDocument(), publicRpcCalls()]); + const missing: string[] = []; + + for (const call of calls) { + const method = protocol.properties.methods.properties[call.wireMethod]; + if (!method) continue; + const fields = Object.entries( + schemaProperties(protocol, resolveSchema(protocol, method.properties.result)), + ) + .filter(([, schema]) => resolveSchema(protocol, schema).const === undefined) + .map(([field]) => field); + if (fields.length === 0 || callReturnsTransportResult(call)) continue; + + const resultName = resultBinding(call); + if (!resultName) continue; + const scopeText = call.scope.text(); + if ( + returnsOrSpreads(scopeText, resultName) || + passesResultWhole(call, resultName) || + assignsResultWhole(call, resultName) + ) { + continue; + } + + for (const field of fields) { + const fieldPath = `${call.wireMethod}.${field}`; + if ( + !usesResultField(scopeText, resultName, field) && + !intentionallyUnusedResultFields.has(fieldPath) + ) { + missing.push(`${call.language} ${call.wireMethod}: result field ${field}`); + } + } + } + + expect( + missing.sort(), + "Every protocol result field must be returned wholesale, spread, or visibly consumed by the public wrapper", + ).toEqual([]); + }); + + it("discovers every RPC-backed TypeScript object from source and gives it a reference page", async () => { + const exportedModules = await typescriptExportedModules(); + const calls = (await publicRpcCalls()).filter( + ({ file, language }) => language === "typescript" && exportedModules.has(file), + ); + const discoveredPages = [ + ...new Set( + calls.map(({ file }) => + snakeCase(basename(file, ".ts")) + .replace(/^browser_/u, "") + .replaceAll("_", "-"), + ), + ), + ].sort(); + const referencePages = (await readdir(referenceUrl)) + .filter((file) => file.endsWith(".mdx")) + .map((file) => basename(file, ".mdx")) + .sort(); + + expect( + discoveredPages, + "Every source file containing a public RPC-backed TypeScript method must have a reference page, and stale reference pages must be removed or classified elsewhere", + ).toStrictEqual(referencePages); + }); +}); + +async function protocolDocument(): Promise { + return JSON.parse(await readFile(protocolUrl, "utf8")) as ProtocolDocument; +} + +function resolveSchema(protocol: ProtocolDocument, schema: JsonSchema): JsonSchema { + if (!schema.$ref) return schema; + const name = schema.$ref.match(/^#\/\$defs\/(.+)$/u)?.[1]; + if (!name || !protocol.$defs[name]) throw new Error(`Unknown schema reference ${schema.$ref}`); + return protocol.$defs[name]; +} + +function schemaProperties( + protocol: ProtocolDocument, + schema: JsonSchema, + seen = new Set(), +): Record { + const resolved = resolveSchema(protocol, schema); + if (seen.has(resolved)) return {}; + const nextSeen = new Set([...seen, resolved]); + return Object.assign( + {}, + resolved.properties ?? {}, + ...(resolved.allOf ?? []).map((part) => schemaProperties(protocol, part, nextSeen)), + ...(resolved.anyOf ?? []).map((part) => schemaProperties(protocol, part, nextSeen)), + ...(resolved.oneOf ?? []).map((part) => schemaProperties(protocol, part, nextSeen)), + ); +} + +function nestedFieldNames(protocol: ProtocolDocument, schema: JsonSchema): string[] { + const properties = schemaProperties(protocol, schema); + return [...new Set(Object.keys(properties))]; +} + +async function sdkSourceFiles(source: URL, language: Language): Promise { + const extension = language === "typescript" ? ".ts" : language === "python" ? ".py" : ".go"; + return (await readdir(source, { recursive: true })) + .filter( + (file) => + file.endsWith(extension) && + !file.endsWith(`_test${extension}`) && + !file.endsWith(`.test${extension}`) && + !file.split("/").includes("tests") && + !file.split("/").includes("_generated"), + ) + .sort(); +} + +async function publicRpcCalls(): Promise { + const [registry, exportedTypescriptModules] = await Promise.all([ + registryNames(), + typescriptExportedModules(), + ]); + const calls = await Promise.all( + (Object.entries(sources) as Array<[Language, URL]>).map(async ([language, source]) => { + const files = await sdkSourceFiles(source, language); + const languageCalls: RpcCall[] = []; + + for (const file of files) { + if (language === "typescript" && !exportedTypescriptModules.has(file)) continue; + const module = parse(language, await readFile(new URL(file, source), "utf8")).root(); + const callKind = language === "python" ? "call" : "call_expression"; + for (const call of module.findAll({ rule: { kind: callKind } })) { + const called = namedChildren(call)[0]?.text(); + const isOutbound = + language === "go" + ? called?.endsWith(".call") === true + : called?.endsWith(".send") === true || called?.endsWith("?.send") === true; + if (!isOutbound) continue; + const arguments_ = callArguments(call); + const methodNode = language === "go" ? arguments_[1] : arguments_[0]; + const params = language === "go" ? arguments_[2] : arguments_[1]; + const result = language === "go" ? arguments_[3] : arguments_[2]; + const wireMethod = wireMethodName(methodNode, language, registry); + const scope = enclosingScope(call, language); + if (!wireMethod || !params || !scope || !isPublicScope(scope, language)) continue; + languageCalls.push({ + call, + file, + language, + module, + params, + result, + scope, + wireMethod, + }); + } + } + return languageCalls; + }), + ); + return calls.flat(); +} + +async function registryNames(): Promise> { + const root = parse("typescript", await readFile(registryUrl, "utf8")).root(); + const declaration = root.find({ rule: { pattern: "const StagehandMethods = $METHODS" } }); + const registry = declaration?.getMatch("METHODS")?.find({ rule: { kind: "object" } }); + if (!registry) throw new Error("StagehandMethods was not found"); + return new Map( + namedChildren(registry).flatMap((entry) => { + if (entry.kind() !== "pair") return []; + const [key, value] = namedChildren(entry); + const name = value + ? namedChildren(value).find( + (property) => + property.kind() === "pair" && namedChildren(property)[0]?.text() === "name", + ) + : undefined; + const wireName = name && namedChildren(name)[1]; + return key && wireName ? [[key.text(), stringLiteral(wireName)] as const] : []; + }), + ); +} + +function wireMethodName( + method: SgNode | undefined, + language: Language, + registry: ReadonlyMap, +): string | undefined { + if (!method) return undefined; + if (language === "typescript") { + if (!method.text().startsWith("StagehandMethods.")) return undefined; + return registry.get(method.text().slice("StagehandMethods.".length)); + } + const expectedKind = language === "python" ? "string" : "interpreted_string_literal"; + return method.kind() === expectedKind ? stringLiteral(method) : undefined; +} + +function enclosingScope(call: SgNode, language: Language): SgNode | undefined { + const kinds = + language === "typescript" + ? new Set(["method_definition", "function_declaration"]) + : language === "python" + ? new Set(["function_definition"]) + : new Set(["method_declaration", "function_declaration"]); + const scopes = call.ancestors().filter((ancestor) => kinds.has(String(ancestor.kind()))); + return language === "python" ? scopes.at(-1) : scopes[0]; +} + +function isPublicScope(scope: SgNode, language: Language): boolean { + const name = scope.field("name")?.text() ?? firstNamedIdentifier(scope)?.text(); + if (!name) return false; + if (language === "python") return !name.startsWith("_"); + if (language === "go") return /^[A-Z]/u.test(name); + if (name === "constructor" || name.startsWith("#")) return false; + const prefix = scope.text().slice(0, scope.text().indexOf(name)); + return !/\b(?:private|protected)\b/u.test(prefix); +} + +function firstNamedIdentifier(node: SgNode): SgNode | undefined { + return node + .findAll({ rule: { kind: "identifier" } }) + .find((identifier) => identifier.text() !== "func" && identifier.text() !== "async"); +} + +function directlyForwardsParameter(params: SgNode, field: string, scopeText: string): boolean { + const normalized = snakeCase(field); + if (new RegExp(`["']?${field}["']?\\s*:\\s*${field}\\b`, "iu").test(scopeText)) { + return true; + } + const occurrences = (params.text().match(/[A-Za-z_][A-Za-z0-9_]*/gu) ?? []) + .map(snakeCase) + .filter((token) => token === normalized).length; + if (occurrences >= 2) return true; + if ( + new RegExp(`(?:[{,]\\s*${field}\\s*[,}]|\\.\\.\\.[^{]*\\{\\s*${field}\\s*[,}])`, "u").test( + params.text(), + ) + ) { + return true; + } + return [ + ...scopeText.matchAll(/["']?([A-Za-z_][A-Za-z0-9_]*)["']?\s*[:=]\s*([A-Za-z_][A-Za-z0-9_]*)/gu), + ].some( + (match) => + snakeCase(match[1] as string) === normalized && snakeCase(match[2] as string) === normalized, + ); +} + +function isWholeParamsReference(params: SgNode): boolean { + return ( + params.kind() === "member_expression" || + params.kind() === "attribute" || + params.kind() === "selector_expression" + ); +} + +function callReturnsTransportResult(call: RpcCall): boolean { + return call.call.ancestors().some((ancestor) => { + if (ancestor.range().start.index < call.scope.range().start.index) return false; + return ancestor.kind() === "return_statement"; + }); +} + +function resultBinding(call: RpcCall): string | undefined { + if (call.language === "go") { + return call.result + ?.text() + .replace(/^&/u, "") + .match(/[A-Za-z_][A-Za-z0-9_]*/u)?.[0]; + } + const assignmentKinds = + call.language === "python" + ? new Set(["assignment"]) + : new Set(["variable_declarator", "assignment_expression"]); + const assignment = call.call + .ancestors() + .find( + (ancestor) => + ancestor.range().start.index >= call.scope.range().start.index && + assignmentKinds.has(String(ancestor.kind())), + ); + if (!assignment) return undefined; + const left = assignment.field("name") ?? assignment.field("left") ?? namedChildren(assignment)[0]; + return left?.text().match(/[A-Za-z_][A-Za-z0-9_]*/u)?.[0]; +} + +function returnsOrSpreads(scopeText: string, resultName: string): boolean { + return ( + new RegExp(`\\breturn(?:\\s+await)?\\s+${resultName}\\b`, "u").test(scopeText) || + new RegExp(`\\.\\.\\.${resultName}\\b`, "u").test(scopeText) + ); +} + +function usesResultField(scopeText: string, resultName: string, field: string): boolean { + return [ + ...scopeText.matchAll(new RegExp(`\\b${resultName}\\.([A-Za-z_][A-Za-z0-9_]*)`, "gu")), + ].some((match) => snakeCase(match[1] as string) === snakeCase(field)); +} + +function passesResultWhole(call: RpcCall, resultName: string): boolean { + const callKind = call.language === "python" ? "call" : "call_expression"; + const calls = [ + ...call.scope.findAll({ rule: { kind: callKind } }), + ...(call.language === "typescript" + ? call.scope.findAll({ rule: { kind: "new_expression" } }) + : []), + ]; + return calls.some((candidate) => { + if (candidate.range().start.index === call.call.range().start.index) return false; + const called = namedChildren(candidate)[0]?.text(); + return ( + called?.startsWith(`${resultName}.`) === true || + callArguments(candidate).some( + (argument) => argument.text().replace(/^[&*]/u, "") === resultName, + ) + ); + }); +} + +function assignsResultWhole(call: RpcCall, resultName: string): boolean { + const text = call.scope.text(); + if ( + new RegExp( + `(?:[.#][A-Za-z_$][A-Za-z0-9_$]*[^\\n=]*?(?:\\?\\?=|=)|\\b[A-Za-z_$][A-Za-z0-9_$]*\\s*:)\\s*[&*]?${resultName}\\b`, + "u", + ).test(text) + ) { + return true; + } + const assignmentKinds = + call.language === "python" + ? ["assignment"] + : call.language === "go" + ? ["assignment_statement"] + : ["assignment_expression"]; + return assignmentKinds.some((kind) => + call.scope.findAll({ rule: { kind } }).some((assignment) => { + const children = namedChildren(assignment); + const left = assignment.field("left") ?? children[0]; + const right = assignment.field("right") ?? children.at(-1); + return left?.text() !== resultName && right?.text().replace(/^&/u, "") === resultName; + }), + ); +} + +async function callableBodies(): Promise>> { + const entries = await Promise.all( + (Object.entries(sources) as Array<[Language, URL]>).map(async ([language, source]) => { + const files = await sdkSourceFiles(source, language); + const bodies = new Map(); + for (const file of files) { + const root = parse(language, await readFile(new URL(file, source), "utf8")).root(); + const kinds = + language === "typescript" + ? ["function_declaration", "method_definition"] + : language === "python" + ? ["function_definition"] + : ["function_declaration", "method_declaration"]; + for (const kind of kinds) { + for (const callable of root.findAll({ rule: { kind } })) { + const name = callable.field("name")?.text() ?? firstNamedIdentifier(callable)?.text(); + if (name) bodies.set(name, `${bodies.get(name) ?? ""}\n${callable.text()}`); + } + } + } + return [language, bodies] as const; + }), + ); + return new Map(entries); +} + +function relatedHelperBodies( + scope: SgNode, + language: Language, + helperBodies: ReadonlyMap>, +): string { + const bodies = helperBodies.get(language); + if (!bodies) return ""; + const callKind = language === "python" ? "call" : "call_expression"; + return scope + .findAll({ rule: { kind: callKind } }) + .flatMap((call) => { + const called = namedChildren(call)[0]?.text().split(".").at(-1)?.replace(/^\?\./u, ""); + const body = called && bodies.get(called); + return body ? [body] : []; + }) + .join("\n"); +} + +async function typescriptExportedModules(): Promise> { + const root = parse( + "typescript", + await readFile(new URL("index.ts", sources.typescript), "utf8"), + ).root(); + const modules = new Set(); + for (const statement of root.findAll({ rule: { kind: "export_statement" } })) { + const source = statement + .findAll({ rule: { kind: "string" } }) + .map(stringLiteral) + .find((value) => value.startsWith("./")); + if (!source) continue; + modules.add(source.replace(/^\.\//u, "").replace(/\.js$/u, ".ts")); + } + return modules; +} + +function semanticTokens(text: string): Set { + return new Set((text.match(/[A-Za-z_][A-Za-z0-9_]*/gu) ?? []).map(snakeCase)); +} + +function snakeCase(value: string): string { + let normalized = value; + for (const acronym of ["HTTPS", "HTTP", "CDP", "API", "URL", "LLM", "MCP", "MIME", "ID"]) { + normalized = normalized.replaceAll(acronym, `_${acronym.toLowerCase()}_`); + } + return normalized + .replace(/([a-z\d])([A-Z])/gu, "$1_$2") + .replace(/([A-Z]+)([A-Z][a-z])/gu, "$1_$2") + .replace(/[-_]+/gu, "_") + .replace(/^_|_$/gu, "") + .toLowerCase(); +} + +function callArguments(call: SgNode): SgNode[] { + const argumentsNode = + call.field("arguments") ?? + namedChildren(call).find( + (child) => child.kind() === "arguments" || child.kind() === "argument_list", + ); + return argumentsNode ? namedChildren(argumentsNode) : []; +} + +function namedChildren(node: SgNode): SgNode[] { + return node.children().filter((child) => child.isNamed()); +} + +function stringLiteral(node: SgNode): string { + return node.text().replace(/^['"`]|['"`]$/gu, ""); +} diff --git a/rules/ast-grep/sdk-parity.test.ts b/rules/ast-grep/sdk-parity.test.ts index 5107f2f33f..17aacc2f9e 100644 --- a/rules/ast-grep/sdk-parity.test.ts +++ b/rules/ast-grep/sdk-parity.test.ts @@ -29,6 +29,7 @@ const sdkObjects = [ const typescriptSource = new URL("../../packages/sdk-ts/src/", import.meta.url); const pythonSource = new URL("../../packages/sdk-python/src/stagehand/", import.meta.url); const goSource = new URL("../../packages/sdk-go/", import.meta.url); +const extensionRouterUrl = new URL("../../packages/extension/rpcRouter.ts", import.meta.url); const protocolUrl = new URL("../../packages/protocol/stagehand.v4.json", import.meta.url); const registryUrl = new URL("../../packages/protocol/schema-registry.ts", import.meta.url); @@ -159,19 +160,56 @@ describe("All language SDK operations remain in sync", () => { const registeredOperations = [...registry.values()].sort(); expect( - await clientProtocolOperations("typescript", typescriptSource, registry), + await protocolOperations("typescript", typescriptSource, registry), "TypeScript must reference every StagehandMethods operation", ).toStrictEqual(registeredOperations); expect( - await clientProtocolOperations("python", pythonSource, registry), + await protocolOperations("python", pythonSource, registry), "Python must reference every StagehandMethods operation", ).toStrictEqual(registeredOperations); expect( - await clientProtocolOperations("go", goSource, registry), + await protocolOperations("go", goSource, registry), "Go must reference every StagehandMethods operation", ).toStrictEqual(registeredOperations); }); + it("routes every protocol operation to exactly one receiving endpoint", async () => { + const registry = await stagehandMethodNames(); + const [extensionInbound, typescriptInbound] = await Promise.all([ + extensionRouterOperations(), + protocolOperations("typescript", typescriptSource, registry, "inbound"), + ]); + const registeredOperations = [...registry.values()].sort(); + const handledByBothEndpoints = extensionInbound.filter((method) => + typescriptInbound.includes(method), + ); + const handledOperations = [...new Set([...extensionInbound, ...typescriptInbound])].sort(); + + expect( + handledByBothEndpoints, + "A protocol operation must not be handled by both the extension and the SDKs", + ).toEqual([]); + expect( + handledOperations, + "Every StagehandMethods operation must have exactly one receiving endpoint", + ).toStrictEqual(registeredOperations); + + for (const [language, source] of [ + ["typescript", typescriptSource], + ["python", pythonSource], + ["go", goSource], + ] as const) { + expect( + await protocolOperations(language, source, registry, "outbound"), + `${language} outbound operations must match the extension router`, + ).toStrictEqual(extensionInbound); + expect( + await protocolOperations(language, source, registry, "inbound"), + `${language} inbound request handlers must match TypeScript inbound request handlers`, + ).toStrictEqual(typescriptInbound); + } + }); + it("keeps every registered notification in the generated protocol and every client", async () => { const [registry, protocol] = await Promise.all([ stagehandNotificationNames(), @@ -712,20 +750,29 @@ async function publicAccessors( .sort(); } -async function clientProtocolOperations( +type RequestBoundary = "inbound" | "outbound"; + +async function protocolOperations( language: SdkLanguage, source: URL, registry: ReadonlyMap, + boundary?: RequestBoundary, ): Promise { const extension = language === "typescript" ? ".ts" : language === "python" ? ".py" : ".go"; const files = (await readdir(source, { recursive: true })) - .filter((file) => file.endsWith(extension) && !file.endsWith(`_test${extension}`)) + .filter( + (file) => + file.endsWith(extension) && + !file.endsWith(`_test${extension}`) && + !file.endsWith(`.test${extension}`) && + !file.split("/").includes("tests"), + ) .sort(); const operations = new Set(); for (const file of files) { const root = parse(language, await readFile(new URL(file, source), "utf8")).root(); - for (const call of protocolCalls(root, language)) { + for (const call of protocolCalls(root, language, boundary)) { const method = protocolMethodNode(call, language); if (method) operations.add(wireMethodForCall(method, language, registry)); } @@ -734,6 +781,24 @@ async function clientProtocolOperations( return [...operations].sort(); } +async function extensionRouterOperations(): Promise { + const root = parse("typescript", await readFile(extensionRouterUrl, "utf8")).root(); + const routeSwitches = root + .findAll({ rule: { kind: "switch_statement" } }) + .filter((statement) => namedChildren(statement)[0]?.text() === "(request.method)"); + if (routeSwitches.length !== 1) { + throw new Error(`Expected one request.method router switch, received ${routeSwitches.length}`); + } + + return routeSwitches[0]! + .findAll({ rule: { kind: "switch_case" } }) + .flatMap((case_) => { + const method = namedChildren(case_)[0]; + return method?.kind() === "string" ? [stringLiteral(method)] : []; + }) + .sort(); +} + async function stagehandMethodNames(): Promise> { return stagehandRegistryNames("StagehandMethods"); } @@ -958,20 +1023,14 @@ async function goRpcCalls(): Promise { return calls; } -function protocolCalls(node: SgNode, language: SdkLanguage): SgNode[] { +function protocolCalls(node: SgNode, language: SdkLanguage, boundary?: RequestBoundary): SgNode[] { const callKind = language === "python" ? "call" : "call_expression"; return node.findAll({ rule: { kind: callKind } }).filter((call) => { const calledFunction = namedChildren(call)[0]?.text(); - const isProtocolBoundary = - calledFunction?.endsWith(".send") === true || - calledFunction?.endsWith("?.send") === true || - (language === "typescript" && - (calledFunction?.endsWith(".onRequest") === true || - calledFunction?.endsWith("?.onRequest") === true)) || - (language === "python" && calledFunction?.endsWith(".on_request") === true) || - (language === "go" && - (calledFunction?.endsWith(".call") === true || - calledFunction?.endsWith(".onRequest") === true)); + const isProtocolBoundary = boundary + ? matchesProtocolBoundary(calledFunction, language, boundary) + : matchesProtocolBoundary(calledFunction, language, "outbound") || + matchesProtocolBoundary(calledFunction, language, "inbound"); if (!isProtocolBoundary) return false; const method = protocolMethodNode(call, language); return language === "typescript" @@ -982,6 +1041,24 @@ function protocolCalls(node: SgNode, language: SdkLanguage): SgNode[] { }); } +function matchesProtocolBoundary( + calledFunction: string | undefined, + language: SdkLanguage, + boundary: RequestBoundary, +): boolean { + if (boundary === "outbound") { + return language === "go" + ? calledFunction?.endsWith(".call") === true + : calledFunction?.endsWith(".send") === true || calledFunction?.endsWith("?.send") === true; + } + return language === "typescript" + ? calledFunction?.endsWith(".onRequest") === true || + calledFunction?.endsWith("?.onRequest") === true + : language === "python" + ? calledFunction?.endsWith(".on_request") === true + : calledFunction?.endsWith(".onRequest") === true; +} + type DirectClassMethod = { node: SgNode; decoratedDefinition?: SgNode; diff --git a/rules/oxlint/no-loose-json-schema-in-protocol-operations.ts b/rules/oxlint/no-loose-json-schema-in-protocol-operations.ts index 2dafb590ce..5030a4227d 100644 --- a/rules/oxlint/no-loose-json-schema-in-protocol-operations.ts +++ b/rules/oxlint/no-loose-json-schema-in-protocol-operations.ts @@ -6,6 +6,33 @@ function keyName(key: ESTree.PropertyKey): string | null { return null; } +function protocolRegistryName(node: ESTree.ObjectProperty): string | null { + const operation = node.parent; + if (operation.type !== "ObjectExpression") return null; + + const operationEntry = operation.parent; + if (operationEntry.type !== "Property" || operationEntry.value !== operation) return null; + + const registry = operationEntry.parent; + if (registry.type !== "ObjectExpression") return null; + + let parent = registry.parent; + while ( + parent.type === "ParenthesizedExpression" || + parent.type === "TSAsExpression" || + parent.type === "TSNonNullExpression" || + parent.type === "TSSatisfiesExpression" || + parent.type === "TSTypeAssertion" + ) { + parent = parent.parent; + } + + if (parent.type !== "VariableDeclarator" || parent.id.type !== "Identifier") return null; + return parent.id.name === "StagehandMethods" || parent.id.name === "StagehandNotifications" + ? parent.id.name + : null; +} + export const noLooseJsonSchemaInProtocolOperations = defineRule({ meta: { type: "problem", @@ -31,8 +58,10 @@ export const noLooseJsonSchemaInProtocolOperations = defineRule({ }, Property(node: ESTree.ObjectProperty) { + if (!protocolRegistryName(node)) return; + const propertyName = keyName(node.key); - if (propertyName !== "paramsSchema" && propertyName !== "resultSchema") return; + if (propertyName !== "params" && propertyName !== "result") return; if (node.value.type === "Identifier" && canonicalSchemas.has(node.value.name)) return; context.report({