fix: make the 2FA take-over universal — tailor guidance to what the page asks for

apps/api/src/browserbase/browser-credential-login.spec.ts

@@ -1,7 +1,6 @@
 import {
   performCredentialLogin,
   reloginWithStoredCredentials,
-  safeOriginAndPath,
 } from './browser-credential-login';
 import type { BrowserCredentialVaultAdapter } from './credential-vault';
 import type { BrowserbaseSessionService } from './browserbase-session.service';
@@ -67,6 +66,61 @@ describe('performCredentialLogin', () => {
       expect(instruction).not.toContain('424242');
     }
   });
+
+  it('switches a passkey prompt to the authenticator-code method when a code is stored', async () => {
+    const stagehand = makeStagehand();
+
+    const promise = performCredentialLogin({
+      stagehand: stagehand as unknown as Stagehand,
+      credentials: { username: 'alice', password: 'pw', totpCode: '424242' },
+      log: jest.fn(),
+    });
+    await jest.runAllTimersAsync();
+    await promise;
+
+    // With a code to fill and no human present, we auto-switch off a passkey so
+    // the code field appears, then enter the stored code. Reaching the code field
+    // can take two clicks, and act() does one thing per call, so it's two steps:
+    // first reveal the other options, then choose the authenticator/code method.
+    const calls = stagehand.act.mock.calls as [string, unknown?][];
+    const revealCall = calls.find(
+      ([instruction]) =>
+        instruction.includes('passkey') && instruction.includes('More options'),
+    );
+    const selectCall = calls.find(([instruction]) =>
+      instruction.includes('authenticator app'),
+    );
+    expect(revealCall).toBeDefined();
+    expect(selectCall).toBeDefined();
+  });
+
+  it('reveals other sign-in methods without choosing one when a passkey blocks a take-over', async () => {
+    const stagehand = makeStagehand();
+
+    const promise = performCredentialLogin({
+      stagehand: stagehand as unknown as Stagehand,
+      credentials: { username: 'alice', password: 'pw' }, // no stored code → human take-over
+      log: jest.fn(),
+    });
+    await jest.runAllTimersAsync();
+    await promise;
+
+    // With no code to fill, a human takes over — we only surface the other
+    // methods and never pick one, so the user chooses what they can complete.
+    const calls = stagehand.act.mock.calls as [string, unknown?][];
+    const revealCall = calls.find(
+      ([instruction]) =>
+        instruction.includes('passkey') &&
+        instruction.includes('More options') &&
+        instruction.includes('do NOT select'),
+    );
+    expect(revealCall).toBeDefined();
+    // And it must NOT pick a method for them (no authenticator-app selection).
+    const forcedSelect = calls.find(([instruction]) =>
+      instruction.includes('authenticator app'),
+    );
+    expect(forcedSelect).toBeUndefined();
+  });
 });
 
 describe('reloginWithStoredCredentials', () => {
@@ -186,24 +240,3 @@ describe('reloginWithStoredCredentials', () => {
     expect(result.reason).toMatch(/user action/i);
   });
 });
-
-describe('safeOriginAndPath (keeps auth secrets out of the LLM prompt)', () => {
-  it('drops the query and fragment (OAuth code/state/tokens)', () => {
-    expect(
-      safeOriginAndPath(
-        'https://login.example.com/callback?code=SECRET&state=xyz#access_token=abc',
-      ),
-    ).toBe('https://login.example.com/callback');
-  });
-
-  it('drops userinfo', () => {
-    expect(safeOriginAndPath('https://user:pass@example.com/app')).toBe(
-      'https://example.com/app',
-    );
-  });
-
-  it('returns empty string for an unparseable or empty URL', () => {
-    expect(safeOriginAndPath('not a url')).toBe('');
-    expect(safeOriginAndPath('')).toBe('');
-  });
-});

apps/api/src/browserbase/browser-credential-login.ts

@@ -1,9 +1,12 @@
-import { z } from 'zod';
 import type {
   BrowserCredentialVaultAdapter,
   RuntimeCredentialMaterial,
 } from './credential-vault';
 import type { BrowserbaseSessionService } from './browserbase-session.service';
+import {
+  classifyLoginOutcome,
+  type SignInOutcome,
+} from './browser-login-classifier';
 
 type Stagehand = import('@browserbasehq/stagehand').Stagehand;
 type ActivePage = Awaited<
@@ -28,21 +31,6 @@ export interface CredentialLoginTarget {
 
 const delay = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
 
-/**
- * Origin + path only — drops the query, fragment, and userinfo so secrets that
- * land in an auth redirect (OAuth `code`/`state`, tokens) are never forwarded to
- * the model. Returns '' for an unparseable URL.
- */
-export function safeOriginAndPath(rawUrl: string): string {
-  if (!rawUrl) return '';
-  try {
-    const url = new URL(rawUrl);
-    return `${url.origin}${url.pathname}`;
-  } catch {
-    return '';
-  }
-}
-
 /**
  * Drives an automated sign-in using stored credentials. Secret values are passed
  * through Stagehand's `variables` substitution, so they are injected into the page
@@ -112,6 +100,20 @@ export async function performCredentialLogin({
   }
 
   if (credentials.totpCode) {
+    // We have a code to enter. Some vendors (e.g. GitHub) default the two-factor
+    // step to a passkey / security key, which we can't use — switch to the
+    // authenticator/code method so a six-digit field appears, then fill it.
+    // Reaching a code field can take two clicks (expand the options, then choose
+    // the code method), and act() does ONE thing per call, so these are separate
+    // steps. Both are best-effort no-ops when a code field is already shown.
+    await stagehand.act(
+      "If this page is asking for a passkey or security key instead of a verification code, click a control that reveals the other sign-in options — 'More options', 'Try another way', 'Use a different method', or similar. If a code field or the list of options is already visible, or this is not a two-factor page, do nothing.",
+    );
+    await delay(1000);
+    await stagehand.act(
+      "If there is an option to verify using an authenticator app or a two-factor code — 'Use authenticator app', 'Enter a two-factor code', 'Use a security code', 'Authenticator app', or similar — click it so a six-digit code field appears. If a code field is already visible, do nothing.",
+    );
+    await delay(1500);
     log('Entering one-time passcode.');
     await stagehand.act(
       'If a one-time passcode, two-factor, or verification code field is shown, enter %code% into it. If no such field is present, do nothing.',
@@ -122,6 +124,16 @@ export async function performCredentialLogin({
       'If there is a button to submit or verify the code, click it. Otherwise do nothing.',
     );
     await delay(2000);
+  } else {
+    // No stored code — if a two-factor step blocks us, a human takes over. Don't
+    // pick a method for them: if the page defaults to a passkey / security key
+    // (which can't be used here), just REVEAL the other options so they can choose
+    // the one they can actually complete (authenticator app, SMS, email). We never
+    // select a method — the choice is theirs. Best-effort no-op otherwise.
+    await stagehand.act(
+      "If this page is asking for a passkey or security key, reveal the other sign-in methods by clicking 'More options', 'Try another way', 'Use a different method', or a similar control — but do NOT select any specific method. If a verification-code field or the list of options is already visible, or this is not a two-factor page, do nothing.",
+    );
+    await delay(1500);
   }
 }
 
@@ -222,66 +234,6 @@ async function runLoginAttempt({
   return sessions.ensureActivePage(stagehand);
 }
 
-export type SignInOutcome =
-  | 'logged_in'
-  | 'invalid_credentials'
-  | 'needs_2fa'
-  | 'challenge'
-  | 'unknown';
-
-/**
- * Reads the current page after a sign-in attempt and classifies the outcome, so
- * the connect flow can tell the user what happened and route them correctly.
- * Never throws — an unreadable page degrades to 'unknown'.
- */
-export async function classifyLoginOutcome(
-  stagehand: Stagehand,
-): Promise<SignInOutcome> {
-  try {
-    // Give the model where the browser actually is, so it can judge for itself
-    // whether we're on the real app or still on a sign-in / identity-provider
-    // page (it knows hosts like signin.aws.amazon.com or login.microsoftonline.com
-    // without us hardcoding URL patterns). A hint only — falls back to content.
-    let currentUrl = '';
-    try {
-      const pages = stagehand.context?.pages?.() ?? [];
-      const rawUrl = pages[pages.length - 1]?.url() ?? '';
-      // Only the origin + path is needed to judge "is this a sign-in page". Strip
-      // the query, fragment, and userinfo so OAuth codes / tokens / state that
-      // land in an auth redirect are never forwarded to the model.
-      currentUrl = safeOriginAndPath(rawUrl);
-    } catch {
-      // URL unavailable — classify from page content alone.
-    }
-    const { state } = await stagehand.extract(
-      (currentUrl ? `The browser is currently at this URL: ${currentUrl}\n\n` : '') +
-        'Classify this page after a sign-in attempt, using BOTH the page content AND the URL. ' +
-        'Return exactly one value: ' +
-        '"logged_in" — there is clear evidence the user is signed in to the actual application ' +
-        '(a dashboard, an account/avatar menu, or a "Sign out" control) AND the browser is on the ' +
-        'application itself. If the URL is a sign-in, login, SSO, or identity-provider page, or the ' +
-        'page is blank, loading, redirecting, or still shows a sign-in form or a "Sign in" / ' +
-        '"Log in" button, it is NOT logged_in; ' +
-        '"invalid_credentials" — it shows an incorrect username/email/password error; ' +
-        '"needs_2fa" — it asks for a two-factor, one-time, authenticator, or verification code; ' +
-        '"challenge" — it shows a CAPTCHA, a "verify it\'s you", a device-approval, or an email/SMS link step; ' +
-        '"unknown" — none of the above clearly apply.',
-      z.object({
-        state: z.enum([
-          'logged_in',
-          'invalid_credentials',
-          'needs_2fa',
-          'challenge',
-          'unknown',
-        ]),
-      }),
-    );
-    return state;
-  } catch {
-    return 'unknown';
-  }
-}
-
 /**
  * The connect flow's first sign-in: fill the stored credentials and classify
  * where we land. Unlike reloginWithStoredCredentials (used by the scheduler),

apps/api/src/browserbase/browser-credential-signin.service.spec.ts

@@ -175,6 +175,24 @@ describe('BrowserCredentialSigninService', () => {
     expect(profiles.markNeedsReauth).toHaveBeenCalledTimes(1);
   });
 
+  it('classifies the verification method for a take-over so the UI can guide the user', async () => {
+    // 1) already-signed-in check → not in; 2) post-sign-in outcome → needs_2fa;
+    // 3) how the page verifies → a passkey the user can switch away from.
+    const extract = jest
+      .fn()
+      .mockResolvedValueOnce({ state: 'unknown' })
+      .mockResolvedValueOnce({ state: 'needs_2fa' })
+      .mockResolvedValueOnce({ method: 'passkey' });
+    const sessions = makeSessions(extract, jest.fn().mockResolvedValue(undefined));
+    const profiles = makeProfiles(profile);
+    withCredentials({ username: 'user@x.com', password: 'secret' }); // no totpCode
+
+    const result = await run(sessions, profiles);
+
+    expect(result.failure).toBe('needs_2fa');
+    expect(result.twoFactorMethod).toBe('passkey');
+  });
+
   it('re-points the live view after the sign-in attempt so a 2FA take-over sees the right tab', async () => {
     const extract = jest
       .fn()

apps/api/src/browserbase/browser-credential-signin.service.ts

@@ -1,10 +1,12 @@
 import { Injectable, Logger, NotFoundException } from '@nestjs/common';
 import { BrowserbaseSessionService } from './browserbase-session.service';
 import { BrowserAuthProfileService } from './browser-auth-profile.service';
+import { signInAndClassify } from './browser-credential-login';
 import {
   classifyLoginOutcome,
-  signInAndClassify,
-} from './browser-credential-login';
+  classifyTwoFactorMethod,
+  type TwoFactorMethod,
+} from './browser-login-classifier';
 import { navigateToSignIn } from './browser-login-navigation';
 import { resolveBrowserCredentialVaultAdapter } from './browser-credential-vault.factory';
 
@@ -46,6 +48,13 @@ export interface AutoSignInResult {
    * (e.g. sites whose root always shows a login form).
    */
   homeUrl?: string;
+  /**
+   * For a take-over (not signed in): HOW the verification step wants the user to
+   * verify, so the UI can give exact guidance — enter a code, switch off a
+   * passkey, or that a passkey-only login can't be automated. Omitted when the
+   * page couldn't be classified.
+   */
+  twoFactorMethod?: TwoFactorMethod;
 }
 
 const FAILURE_REASON: Record<AutoSignInFailure, string> = {
@@ -249,7 +258,16 @@ export class BrowserCredentialSigninService {
       });
       await record(outcome, FAILURE_REASON[outcome]);
       finish('warn');
-      return { isLoggedIn: false, failure: outcome };
+
+      // A human is about to take over a verification step — read HOW the page
+      // wants them to verify (enter a code, switch off a passkey, or a
+      // passkey-only login we can't automate) so the UI gives exact guidance.
+      // Best-effort: undefined just falls back to the generic take-over copy.
+      let twoFactorMethod: TwoFactorMethod | undefined;
+      if (outcome === 'needs_2fa' || outcome === 'challenge') {
+        twoFactorMethod = await classifyTwoFactorMethod(activeStagehand);
+      }
+      return { isLoggedIn: false, failure: outcome, twoFactorMethod };
     } catch (error) {
       // An unexpected error (session/navigation/model failure) is exactly the
       // kind of thing a "can't connect" ticket is about — record it, then let

apps/api/src/browserbase/browser-login-classifier.spec.ts

@@ -0,0 +1,59 @@
+import {
+  classifyTwoFactorMethod,
+  safeOriginAndPath,
+} from './browser-login-classifier';
+
+type Stagehand = import('@browserbasehq/stagehand').Stagehand;
+
+describe('classifyTwoFactorMethod', () => {
+  const makeStagehandWithExtract = (result: unknown) =>
+    ({ extract: jest.fn().mockResolvedValue(result) }) as unknown as Stagehand;
+
+  it('returns the classified method for each valid response', async () => {
+    for (const method of [
+      'code',
+      'passkey',
+      'passkey_only',
+      'other',
+    ] as const) {
+      await expect(
+        classifyTwoFactorMethod(makeStagehandWithExtract({ method })),
+      ).resolves.toBe(method);
+    }
+  });
+
+  it('degrades to "other" for a non-conforming or missing value', async () => {
+    // Wrong shape (e.g. an outcome payload) — must not leak undefined.
+    await expect(
+      classifyTwoFactorMethod(makeStagehandWithExtract({ state: 'needs_2fa' })),
+    ).resolves.toBe('other');
+  });
+
+  it('degrades to "other" when extraction throws', async () => {
+    const stagehand = {
+      extract: jest.fn().mockRejectedValue(new Error('boom')),
+    } as unknown as Stagehand;
+    await expect(classifyTwoFactorMethod(stagehand)).resolves.toBe('other');
+  });
+});
+
+describe('safeOriginAndPath (keeps auth secrets out of the LLM prompt)', () => {
+  it('drops the query and fragment (OAuth code/state/tokens)', () => {
+    expect(
+      safeOriginAndPath(
+        'https://login.example.com/callback?code=SECRET&state=xyz#access_token=abc',
+      ),
+    ).toBe('https://login.example.com/callback');
+  });
+
+  it('drops userinfo', () => {
+    expect(safeOriginAndPath('https://user:pass@example.com/app')).toBe(
+      'https://example.com/app',
+    );
+  });
+
+  it('returns empty string for an unparseable or empty URL', () => {
+    expect(safeOriginAndPath('not a url')).toBe('');
+    expect(safeOriginAndPath('')).toBe('');
+  });
+});