refactor!: Introduce IBrowserPool interface and utilize it in BrowserCrawler (#3669)

Author: janbuchar

docs/upgrading/upgrading_v4.md

@@ -214,6 +214,70 @@ const crawler = new BasicCrawler({
 });
 ```
 
+## Custom `BrowserPool` implementations via the `IBrowserPool` interface
+
+Browser crawlers now accept any object implementing the new `IBrowserPool` interface as their `browserPool` option, not just instances of the built-in `BrowserPool`. The interface follows the classic acquire/release pattern, plus a pair of helpers for moving state between the crawling session and the page:
+
+- **`newPage(options?)`** — opens a new page. An optional `session` can be passed as a best-effort hint — the pool may use it for proxy configuration, fingerprinting, etc., but nothing is guaranteed.
+- **`closePage(page, options?)`** — signals the pool that the caller is done with the page. If the optional `error` is a `SessionError`, the pool should purge all state associated with the session (e.g. retire the underlying browser).
+- **`extractPageState(page)`** — reads the relevant state (currently cookies) out of a page so the crawler can persist it back into the session.
+- **`injectPageState(page, state)`** — the counterpart to `extractPageState`; seeds a page with state (currently cookies) before navigation. Isolation between pages is best-effort and depends on the pool implementation.
+
+Lifecycle (`destroy`) is the responsibility of whoever owns the pool: a custom pool you construct yourself is never owned by the crawler, so the crawler never tears it down. This makes it straightforward to plug in a remote browser farm, a session-aware pool, or another custom browser-management strategy without subclassing `BrowserPool`.
+
+```typescript
+import { PuppeteerCrawler } from '@crawlee/puppeteer';
+import { BrowserPool, PuppeteerPlugin, type IBrowserPool } from '@crawlee/browser-pool';
+import puppeteer from 'puppeteer';
+
+const sharedPool = new BrowserPool({ browserPlugins: [new PuppeteerPlugin(puppeteer)] });
+
+const crawler = new PuppeteerCrawler({
+    browserPool: sharedPool,
+    requestHandler: async ({ page }) => {
+        // …
+    },
+});
+
+// You own `sharedPool` — destroy it yourself when you're done.
+await crawler.run();
+await sharedPool.destroy();
+```
+
+## `BrowserCrawlingContext.browserController` has been removed
+
+The `browserController` property is no longer part of the crawling context (`BrowserCrawlingContext`). Browser controller management is now fully internal to the pool — the crawler interacts with the pool only through the `IBrowserPool` interface (`newPage`, `closePage`, `extractPageState`, and `injectPageState`).
+
+If you previously used `browserController` in your request handlers, here is how to migrate the most common patterns:
+
+**Cookies** — Cookie injection and persistence are now handled automatically by the crawler and the pool. You no longer need to call `browserController.getCookies()` or `browserController.setCookies()` manually.
+
+**Proxy info** — Access proxy information via `session.proxyInfo` instead of `browserController.launchContext.proxyUrl`. TLS-error handling moved along with it: the pool reads `session.proxyInfo.ignoreTlsErrors`, so there is no standalone `ignoreTlsErrors` page option anymore. If you need to disable TLS verification for some other reason, set `ignoreHTTPSErrors` (Playwright) / `acceptInsecureCerts` (Puppeteer) through the browser's `launchOptions`.
+
+**Direct browser access** — If you need the raw browser or controller instance (e.g. for Puppeteer/Playwright-specific APIs), construct a `BrowserPool` yourself, pass it to the crawler, and reference it directly in your handler — no cast needed:
+
+```typescript
+import { BrowserPool, PuppeteerPlugin } from '@crawlee/browser-pool';
+import { PuppeteerCrawler } from '@crawlee/puppeteer';
+import puppeteer from 'puppeteer';
+
+const pool = new BrowserPool({ browserPlugins: [new PuppeteerPlugin(puppeteer)] });
+
+const crawler = new PuppeteerCrawler({
+    browserPool: pool,
+    requestHandler: async ({ page }) => {
+        const controller = pool.getBrowserControllerByPage(page);
+        // controller.browser, controller.launchContext, etc.
+    },
+});
+
+await crawler.run();
+// You own the pool — tear it down yourself.
+await pool.destroy();
+```
+
+Note that this couples your code to the built-in `BrowserPool` — custom `IBrowserPool` implementations may not expose controllers at all.
+
 ## `tieredProxyUrls` is removed from `ProxyConfiguration`
 
 The `tieredProxyUrls` option has been removed, together with the `proxyTier` field on `ProxyInfo` and the `proxyTier` plumbing in `BrowserPool`. In v4 the `Session` is the main rotation unit - a session already carries its own proxy, cookies and error score, so the pool rotates the whole fingerprint when a session gets retired on a block.

packages/browser-crawler/src/internals/browser-crawler.ts

@@ -38,7 +38,7 @@ import type {
     LaunchContext,
 } from '@crawlee/browser-pool';
 import { BrowserPool } from '@crawlee/browser-pool';
-import type { BatchAddRequestsResult, Cookie as CookieObject, ISession } from '@crawlee/types';
+import type { BatchAddRequestsResult, Cookie as CookieObject, IBrowserPool, ISession } from '@crawlee/types';
 import type { RobotsTxtFile } from '@crawlee/utils';
 import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';
 import ow from 'ow';
@@ -57,15 +57,9 @@ type ContextDifference<T, U> = Omit<U, keyof T> & Partial<U>;
 export interface BrowserCrawlingContext<
     Page extends CommonPage = CommonPage,
     Response extends BaseResponse = BaseResponse,
-    ProvidedController = BrowserController,
     UserData extends Dictionary = Dictionary,
     GoToOptions extends Dictionary = Dictionary,
 > extends CrawlingContext<UserData> {
-    /**
-     * An instance of the {@apilink BrowserController} that manages the browser instance and provides access to its API.
-     */
-    browserController: ProvidedController;
-
     /**
      * The browser page object where the web page is loaded and rendered.
      */
@@ -104,11 +98,9 @@ const readContextField = <T>(ctx: object, key: symbol): T => (ctx as Record<symb
 export interface BrowserCrawlerOptions<
     Page extends CommonPage = CommonPage,
     Response extends BaseResponse = BaseResponse,
-    ProvidedController extends BrowserController = BrowserController,
-    Context extends BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary> = BrowserCrawlingContext<
+    Context extends BrowserCrawlingContext<Page, Response, Dictionary> = BrowserCrawlingContext<
         Page,
         Response,
-        ProvidedController,
         Dictionary
     >,
     ContextExtension = Dictionary<never>,
@@ -124,6 +116,13 @@ export interface BrowserCrawlerOptions<
 > {
     launchContext?: BrowserLaunchContext<any, any>;
 
+    /**
+     * An existing browser pool instance to use. When provided, the crawler will use this pool directly instead of
+     * constructing a new one from `browserPoolOptions`, enabling browser sharing across multiple crawlers. The crawler
+     * will not tear down a shared pool — the caller is responsible for its lifecycle.
+     */
+    browserPool?: IBrowserPool<Page>;
+
     /**
      * Function that is called to process each request.
      *
@@ -134,7 +133,6 @@ export interface BrowserCrawlerOptions<
      * - {@apilink BrowserCrawlingContext.page|`page`} is an instance of the
      * Puppeteer [Page](https://pptr.dev/api/puppeteer.page) or
      * Playwright [Page](https://playwright.dev/docs/api/class-page);
-     * - {@apilink BrowserCrawlingContext.browserController|`browserController`} is an instance of the {@apilink BrowserController};
      * - {@apilink BrowserCrawlingContext.response|`response`} is an instance of the
      * Puppeteer [Response](https://pptr.dev/api/puppeteer.httpresponse) or
      * Playwright [Response](https://playwright.dev/docs/api/class-response),
@@ -306,23 +304,30 @@ export interface BrowserCrawlerOptions<
 export abstract class BrowserCrawler<
     Page extends CommonPage = CommonPage,
     Response extends BaseResponse = BaseResponse,
-    ProvidedController extends BrowserController = BrowserController,
     InternalBrowserPoolOptions extends BrowserPoolOptions = BrowserPoolOptions,
     LaunchOptions extends Dictionary | undefined = Dictionary,
-    Context extends BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary> = BrowserCrawlingContext<
+    Context extends BrowserCrawlingContext<Page, Response, Dictionary> = BrowserCrawlingContext<
         Page,
         Response,
-        ProvidedController,
         Dictionary
     >,
     ContextExtension = Dictionary<never>,
     ExtendedContext extends Context = Context & ContextExtension,
     GoToOptions extends Dictionary = Dictionary,
 > extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
     /**
-     * A reference to the underlying {@apilink BrowserPool} class that manages the crawler's browsers.
+     * A reference to the underlying browser pool that manages the crawler's browsers. Typed as
+     * {@apilink IBrowserPool} so custom implementations can be plugged in via the `browserPool` constructor option.
      */
-    browserPool: BrowserPool<InternalBrowserPoolOptions>;
+    browserPool: IBrowserPool<Page>;
+
+    /**
+     * Set when the crawler constructed its own {@apilink BrowserPool} (no `browserPool` option was provided).
+     * Holds the same instance as `browserPool`, but typed as the concrete class so the crawler can call
+     * lifecycle methods (`destroy`) that aren't part of {@apilink IBrowserPool}. A user-supplied pool is
+     * never owned and never torn down by the crawler.
+     */
+    private ownedBrowserPool?: BrowserPool<InternalBrowserPoolOptions>;
 
     launchContext: BrowserLaunchContext<LaunchOptions, unknown>;
 
@@ -343,7 +348,8 @@ export abstract class BrowserCrawler<
 
         launchContext: ow.optional.object,
         headless: ow.optional.any(ow.boolean, ow.string),
-        browserPoolOptions: ow.object,
+        browserPool: ow.optional.object.validate(validators.browserPool),
+        browserPoolOptions: ow.optional.object,
         saveResponseCookies: ow.optional.boolean,
         proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
     };
@@ -352,14 +358,7 @@ export abstract class BrowserCrawler<
      * All `BrowserCrawler` parameters are passed via an options object.
      */
     protected constructor(
-        options: BrowserCrawlerOptions<
-            Page,
-            Response,
-            ProvidedController,
-            Context,
-            ContextExtension,
-            ExtendedContext
-        > & {
+        options: BrowserCrawlerOptions<Page, Response, Context, ContextExtension, ExtendedContext> & {
             contextPipelineBuilder: () => ContextPipeline<CrawlingContext, Context>;
         },
     ) {
@@ -368,6 +367,7 @@ export abstract class BrowserCrawler<
             navigationTimeoutSecs = 60,
             saveResponseCookies = true,
             launchContext = {},
+            browserPool,
             browserPoolOptions,
             preNavigationHooks = [],
             postNavigationHooks = [],
@@ -422,43 +422,46 @@ export abstract class BrowserCrawler<
 
         this.saveResponseCookies = saveResponseCookies;
 
+        if (browserPool) {
+            this.browserPool = browserPool;
+            return;
+        }
+
+        const resolvedBrowserPoolOptions = browserPoolOptions ?? ({} as Partial<BrowserPoolOptions>);
+
         if (launchContext?.userAgent) {
-            if (browserPoolOptions.useFingerprints)
+            if (resolvedBrowserPoolOptions.useFingerprints)
                 this.log.info('Custom user agent provided, disabling automatic browser fingerprint injection!');
-            browserPoolOptions.useFingerprints = false;
+            resolvedBrowserPoolOptions.useFingerprints = false;
         }
 
-        this.browserPool = new BrowserPool<InternalBrowserPoolOptions>({
-            ...(browserPoolOptions as any),
+        this.ownedBrowserPool = new BrowserPool<InternalBrowserPoolOptions>({
+            ...(resolvedBrowserPoolOptions as any),
         });
+        this.browserPool = this.ownedBrowserPool as IBrowserPool<Page>;
     }
 
     protected override buildContextPipeline(): ContextPipeline<
         CrawlingContext,
-        BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary>
+        BrowserCrawlingContext<Page, Response, Dictionary>
     > {
         return ContextPipeline.create<CrawlingContext>().compose({
             action: this.preparePage.bind(this),
             cleanup: async (context: {
                 page: Page;
                 session: ISession;
-                browserController: ProvidedController;
                 registerDeferredCleanup: BasicCrawlingContext['registerDeferredCleanup'];
             }) => {
                 context.registerDeferredCleanup(async () => {
-                    // In non-incognito mode the browser controller carries the session's cookies
-                    // and storage across pages. If the session is no longer usable, retire the
-                    // controller so its state can't leak into whichever session lands on it next.
-                    if (!context.session.isUsable()) {
-                        this.browserPool.retireBrowserController(
-                            context.browserController as Parameters<
-                                BrowserPool<InternalBrowserPoolOptions>['retireBrowserController']
-                            >[0],
+                    const error = !context.session.isUsable()
+                        ? new SessionError('Session is no longer usable')
+                        : undefined;
+
+                    await this.browserPool
+                        .closePage(context.page, { error })
+                        .catch((closeError: Error) =>
+                            this.log.debug('Error while closing page', { error: closeError }),
                         );
-                    }
-                    await context.page
-                        .close()
-                        .catch((error: Error) => this.log.debug('Error while closing page', { error }));
                 });
             },
         });
@@ -473,9 +476,7 @@ export abstract class BrowserCrawler<
         return foundSelectors.length > 0 ? foundSelectors : null;
     }
 
-    protected async isRequestBlocked(
-        crawlingContext: BrowserCrawlingContext<Page, Response, ProvidedController>,
-    ): Promise<string | false> {
+    protected async isRequestBlocked(crawlingContext: BrowserCrawlingContext<Page, Response>): Promise<string | false> {
         const { page, response } = crawlingContext;
 
         // Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
@@ -500,27 +501,13 @@ export abstract class BrowserCrawler<
 
     private async preparePage(
         crawlingContext: CrawlingContext,
-    ): Promise<
-        ContextDifference<CrawlingContext, BrowserCrawlingContext<Page, Response, ProvidedController, Dictionary>>
-    > {
-        const newPageOptions: Dictionary = {
+    ): Promise<ContextDifference<CrawlingContext, BrowserCrawlingContext<Page, Response, Dictionary>>> {
+        const page = await this.browserPool.newPage({
             id: crawlingContext.id,
-        };
-
-        if (crawlingContext.session?.proxyInfo) {
-            const proxyInfo = crawlingContext.session.proxyInfo;
-
-            newPageOptions.proxyUrl = proxyInfo?.url;
-            newPageOptions.ignoreTlsErrors = proxyInfo?.ignoreTlsErrors;
-        }
-
-        const page = (await this.browserPool.newPage(newPageOptions)) as Page;
+            session: crawlingContext.session,
+        });
         tryCancel();
 
-        const browserControllerInstance = this.browserPool.getBrowserControllerByPage(
-            page as any,
-        ) as ProvidedController;
-
         const contextEnqueueLinks = crawlingContext.enqueueLinks;
 
         return {
@@ -533,7 +520,6 @@ export abstract class BrowserCrawler<
             get gotoOptions(): Dictionary {
                 throw new Error('The `gotoOptions` property is not available until `prepareNavigation` runs.');
             },
-            browserController: browserControllerInstance,
             enqueueLinks: async (enqueueOptions: EnqueueLinksOptions = {}) => {
                 return (await browserCrawlerEnqueueLinks({
                     options: {
@@ -622,7 +608,7 @@ export abstract class BrowserCrawler<
 
         // TODO: Should we save the cookies also after/only the handle page?
         if (this.saveResponseCookies && crawlingContext.session) {
-            const cookies = await crawlingContext.browserController.getCookies(crawlingContext.page);
+            const { cookies } = await this.browserPool.extractPageState(crawlingContext.page);
             tryCancel();
             const url = crawlingContext.request.loadedUrl!;
             for (const cookie of cookies) {
@@ -639,9 +625,7 @@ export abstract class BrowserCrawler<
         return { request: crawlingContext.request as LoadedRequest<Request> } as Partial<Context>;
     }
 
-    private async handleBlockedRequestByContent(
-        crawlingContext: BrowserCrawlingContext<Page, Response, ProvidedController>,
-    ) {
+    private async handleBlockedRequestByContent(crawlingContext: BrowserCrawlingContext<Page, Response>) {
         if (this.retryOnBlocked) {
             const error = await this.isRequestBlocked(crawlingContext);
             if (error) throw new SessionError(error);
@@ -656,20 +640,19 @@ export abstract class BrowserCrawler<
     }
 
     protected async _applyCookies(
-        { session, request, page, browserController }: BrowserCrawlingContext,
+        { session, request, page }: BrowserCrawlingContext<Page, Response>,
         preHooksCookies: string,
         postHooksCookies: string,
     ) {
         const sessionCookie = session?.cookieJar.getCookiesSync(request.url).map(toughCookieToBrowserPoolCookie) ?? [];
         const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
         const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
 
-        await browserController.setCookies(
-            page,
-            [...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
-                .filter((c): c is CookieObject => typeof c !== 'undefined' && c !== null)
-                .map((c) => ({ ...c, url: c.domain ? undefined : request.url })),
-        );
+        const cookies = [...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
+            .filter((c): c is CookieObject => typeof c !== 'undefined' && c !== null)
+            .map((c) => ({ ...c, url: c.domain ? undefined : request.url }));
+
+        await this.browserPool.injectPageState(page, { cookies });
     }
 
     /**
@@ -695,7 +678,7 @@ export abstract class BrowserCrawler<
     }
 
     protected abstract _navigationHandler(
-        crawlingContext: BrowserCrawlingContext<Page, Response, ProvidedController>,
+        crawlingContext: BrowserCrawlingContext<Page, Response>,
         gotoOptions: GoToOptions,
     ): Promise<Context['response'] | null | undefined>;
 
@@ -735,7 +718,7 @@ export abstract class BrowserCrawler<
      * @ignore
      */
     override async teardown(): Promise<void> {
-        await this.browserPool.destroy();
+        await this.ownedBrowserPool?.destroy();
         await super.teardown();
     }
 }