From d3518808b7da481050bcb86933e7589e760aa0f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Wed, 15 Apr 2026 13:03:08 +0200 Subject: [PATCH 1/2] fix(puppeteer-crawler): migrate cookie handling to browser-context API Puppeteer's page-level cookie API (page.cookies / page.setCookie) is deprecated; the successor is the browser-context level API. Update PuppeteerController to use page.browserContext().cookies() and browserContext().setCookie(), mirroring what the Playwright controller already does. This is a semantics change: the page-level API returned cookies scoped to the page's current URL, the context API returns every cookie in the browser context. With Crawlee's default one-context-per-session setup the effect is invisible, but sessions that share a context will see state bleed between their pages. Documented in docs/upgrading/upgrading_v4.md. Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/upgrading/upgrading_v4.md | 12 ++++++++++++ .../src/puppeteer/puppeteer-controller.ts | 4 ++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md index 157071431ff1..c2e6aa6dd5b6 100644 --- a/docs/upgrading/upgrading_v4.md +++ b/docs/upgrading/upgrading_v4.md @@ -266,3 +266,15 @@ The `transformRequestFunction` callback receives a `RequestOptions` object and c - A new `RequestOptions` plain object - `'unchanged'` to keep the original options as-is - A falsy value or `'skip'` to exclude the request from the queue + +## Puppeteer cookies are now read and written at the browser-context level + +The `PuppeteerController._getCookies` / `_setCookies` methods (used internally by the session pool to sync cookies between a `Session` and a Puppeteer page) now call `page.browserContext().cookies()` / `setCookie()` instead of the deprecated `page.cookies()` / `page.setCookie()`. The page-level API was removed in newer Puppeteer releases. + +This aligns the Puppeteer controller with the Playwright controller, which has always worked at the context level. + +**What changes in practice** +- Cookie reads return every cookie stored in the page's browser context, not just cookies matching the page's current URL. If your `Session` relied on the URL-scoped filtering (for example, to avoid pulling cookies that belong to other tabs in the same context), you'll now see the full set. +- Cookie writes are applied to the whole browser context. When you launch pages with shared contexts, cookies written via `Session.setCookiesFromResponse` or similar will be visible to every other page in that context. + +If you rely on Crawlee's default configuration (one browser context per session, which is the `useIncognitoPages` / `newContextPerSession` behavior used by `PuppeteerCrawler`), you should not notice any difference — each session already owns its own context. diff --git a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts index 9b422b972b11..6c415d4cd43b 100644 --- a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts +++ b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts @@ -140,10 +140,10 @@ export class PuppeteerController extends BrowserController< } protected async _getCookies(page: PuppeteerTypes.Page): Promise { - return page.cookies(); + return page.browserContext().cookies(); } protected async _setCookies(page: PuppeteerTypes.Page, cookies: Cookie[]): Promise { - return page.setCookie(...cookies); + return page.browserContext().setCookie(...(cookies as PuppeteerTypes.CookieData[])); } } From 9daae96126092236c786fadfe6c88b622d4e1f25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Ad=C3=A1mek?= Date: Wed, 15 Apr 2026 13:56:02 +0200 Subject: [PATCH 2/2] fix(puppeteer-crawler): preserve page-scoped url back-fill in _setCookies page.setCookie() used to auto-fill the cookie's `url` with the page's current URL when both `url` and `domain` were missing. BrowserContext.setCookie() doesn't, and Chromium rejects cookies with neither. Replicate the old behavior so callers that omit both fields keep working, and document the caveat in the upgrading guide. Co-Authored-By: Claude Opus 4.6 (1M context) --- docs/upgrading/upgrading_v4.md | 2 ++ .../browser-pool/src/puppeteer/puppeteer-controller.ts | 8 +++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/upgrading/upgrading_v4.md b/docs/upgrading/upgrading_v4.md index c2e6aa6dd5b6..c61c285019d2 100644 --- a/docs/upgrading/upgrading_v4.md +++ b/docs/upgrading/upgrading_v4.md @@ -278,3 +278,5 @@ This aligns the Puppeteer controller with the Playwright controller, which has a - Cookie writes are applied to the whole browser context. When you launch pages with shared contexts, cookies written via `Session.setCookiesFromResponse` or similar will be visible to every other page in that context. If you rely on Crawlee's default configuration (one browser context per session, which is the `useIncognitoPages` / `newContextPerSession` behavior used by `PuppeteerCrawler`), you should not notice any difference — each session already owns its own context. + +**Cookie `url` field** — the old `page.setCookie()` auto-filled a missing `url` on each cookie with the page's current URL. The new `browserContext().setCookie()` does not; Chromium rejects cookies that carry neither `url` nor `domain`. Crawlee's internal `_setCookies` keeps the old behavior by back-filling `page.url()` for any cookie that has neither field set, but if you call `browserContext().setCookie()` directly (outside of Crawlee) you need to provide one of them yourself. diff --git a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts index 6c415d4cd43b..26c1d3fb8f07 100644 --- a/packages/browser-pool/src/puppeteer/puppeteer-controller.ts +++ b/packages/browser-pool/src/puppeteer/puppeteer-controller.ts @@ -144,6 +144,12 @@ export class PuppeteerController extends BrowserController< } protected async _setCookies(page: PuppeteerTypes.Page, cookies: Cookie[]): Promise { - return page.browserContext().setCookie(...(cookies as PuppeteerTypes.CookieData[])); + // BrowserContext.setCookie requires `url` or `domain`; the page-level API used to back-fill + // the page's current URL for us. Replicate that so callers who pass neither don't get rejected. + const pageUrl = page.url(); + const normalized = cookies.map((cookie) => + cookie.url || cookie.domain ? cookie : { ...cookie, url: pageUrl }, + ); + return page.browserContext().setCookie(...(normalized as PuppeteerTypes.CookieData[])); } }