feat: allow navigation hooks to override crawling context members (#3672)

`preNavigationHooks` and `postNavigationHooks` may now return a
`Partial<Context>`; returned own properties are merged into the crawling
context.

Closes #3660

Author: barjin

docs/guides/avoid_blocking.mdx

@@ -58,7 +58,7 @@ On the contrary, sometimes we want to entirely disable the usage of browser fing
 
 ## Camoufox
 
-For some protections, using our integrated solutions is not enough, one example could be the Cloudflare challenge. For such pages, you can try [Camoufox](https://camoufox.com/), a custom stealthy build of Firefox for web scraping. It might not get you through the challenge automatically, but with our `handleCloudflareChallenge` helper, it should be able to successfully mimic the required user action and get you through it.
+For some protections, using our integrated solutions is not enough, one example could be the Cloudflare challenge. For such pages, you can try [Camoufox](https://camoufox.com/), a custom stealthy build of Firefox for web scraping. It might not get you through the challenge automatically, but with our `handleCloudflareChallengeHook` post-navigation hook, it should be able to successfully mimic the required user action and get you through it. The hook also reloads the page after the challenge clears and propagates the fresh response back into the crawling context.
 
 <CodeBlock language="ts">
     {PlaywrightCamoufox}

docs/guides/avoid_blocking_camoufox.ts

@@ -1,13 +1,9 @@
-import { PlaywrightCrawler } from 'crawlee';
+import { PlaywrightCrawler, handleCloudflareChallengeHook } from 'crawlee';
 import { launchOptions } from 'camoufox-js';
 import { firefox } from 'playwright';
 
 const crawler = new PlaywrightCrawler({
-    postNavigationHooks: [
-        async ({ handleCloudflareChallenge }) => {
-            await handleCloudflareChallenge();
-        },
-    ],
+    postNavigationHooks: [handleCloudflareChallengeHook()],
     browserPoolOptions: {
         // Disable the default fingerprint spoofing to avoid conflicts with Camoufox.
         useFingerprints: false,

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -2239,17 +2239,6 @@ export class BasicCrawler<
         }
     }
 
-    protected async _executeHooks<HookLike extends (...args: any[]) => Awaitable<void>>(
-        hooks: HookLike[],
-        ...args: Parameters<HookLike>
-    ) {
-        if (Array.isArray(hooks) && hooks.length) {
-            for (const hook of hooks) {
-                await hook(...args);
-            }
-        }
-    }
-
     /**
      * Stops the crawler immediately.
      *

packages/browser-crawler/src/internals/browser-crawler.ts

@@ -2,6 +2,7 @@ import type {
     Awaitable,
     BasicCrawlerOptions,
     BasicCrawlingContext,
+    ContextMiddleware,
     CrawlingContext,
     Dictionary,
     EnqueueLinksOptions,
@@ -58,6 +59,7 @@ export interface BrowserCrawlingContext<
     Response extends BaseResponse = BaseResponse,
     ProvidedController = BrowserController,
     UserData extends Dictionary = Dictionary,
+    GoToOptions extends Dictionary = Dictionary,
 > extends CrawlingContext<UserData> {
     /**
      * An instance of the {@apilink BrowserController} that manages the browser instance and provides access to its API.
@@ -79,16 +81,25 @@ export interface BrowserCrawlingContext<
      */
     response: Response;
 
+    /**
+     * Options object passed to the underlying `page.goto()` call. `preNavigationHooks` can mutate this
+     * object (or return `{ gotoOptions: ... }`) to influence the navigation.
+     */
+    gotoOptions: GoToOptions;
+
     /**
      * Helper function for extracting URLs from the current page and adding them to the request queue.
      */
     enqueueLinks: (options?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
 }
 
-export type BrowserHook<Context = BrowserCrawlingContext, GoToOptions extends Dictionary | undefined = Dictionary> = (
+export type BrowserHook<Context = BrowserCrawlingContext> = (
     crawlingContext: Context,
-    gotoOptions: GoToOptions,
-) => Awaitable<void>;
+) => Awaitable<void | Partial<Context>>;
+
+const COOKIES_BEFORE_HOOKS = Symbol('cookiesBeforeHooks');
+
+const readContextField = <T>(ctx: object, key: symbol): T => (ctx as Record<symbol, unknown>)[key] as T;
 
 export interface BrowserCrawlerOptions<
     Page extends CommonPage = CommonPage,
@@ -174,31 +185,33 @@ export interface BrowserCrawlerOptions<
 
     /**
      * Async functions that are sequentially evaluated before the navigation. Good for setting additional cookies
-     * or browser properties before navigation. The function accepts two parameters, `crawlingContext` and `gotoOptions`,
-     * which are passed to the `page.goto()` function the crawler calls to navigate.
+     * or browser properties before navigation. The function receives the `crawlingContext`; the options object
+     * forwarded to `page.goto()` is available as `crawlingContext.gotoOptions` and can be mutated in place.
      *
      * **Example:**
      *
      * ```js
      * preNavigationHooks: [
-     *     async (crawlingContext, gotoOptions) => {
-     *         const { page } = crawlingContext;
+     *     async ({ page, gotoOptions }) => {
      *         await page.evaluate((attr) => { window.foo = attr; }, 'bar');
      *         gotoOptions.timeout = 60_000;
      *         gotoOptions.waitUntil = 'domcontentloaded';
      *     },
      * ]
      * ```
      *
-     * Modyfing `pageOptions` is supported only in Playwright incognito.
-     * See {@apilink PrePageCreateHook}
+     * A hook may optionally return a partial object whose properties are merged into the crawling context,
+     * allowing the hook to override context members for subsequent hooks and pipeline stages.
      */
     preNavigationHooks?: BrowserHook<Context>[];
 
     /**
      * Async functions that are sequentially evaluated after the navigation. Good for checking if the navigation was successful.
      * The function accepts `crawlingContext` as the only parameter.
      *
+     * A hook may optionally return a partial object whose properties are merged into the crawling context.
+     * This is useful for overriding context members (e.g. `response`) after solving a challenge.
+     *
      * **Example:**
      *
      * ```js
@@ -209,6 +222,11 @@ export interface BrowserCrawlerOptions<
      *             await solveCaptcha(page);
      *         }
      *     },
+     *     async (crawlingContext) => {
+     *         if (await needsRevalidation(crawlingContext)) {
+     *             return { response: await crawlingContext.page.reload() };
+     *         }
+     *     },
      * ]
      * ```
      */
@@ -358,13 +376,32 @@ export abstract class BrowserCrawler<
             ...basicCrawlerOptions
         } = options;
 
+        const skipGuard = <Ctx extends Context>(
+            action: (ctx: Ctx) => Awaitable<void | Partial<Ctx>>,
+        ): ContextMiddleware<Ctx, Partial<Ctx>> => ({
+            action: async (ctx) => (ctx.request.skipNavigation ? {} : ((await action(ctx)) ?? {})),
+        });
+
         super({
             ...basicCrawlerOptions,
-            contextPipelineBuilder: () =>
-                contextPipelineBuilder()
-                    .compose({ action: this.performNavigation.bind(this) })
+            contextPipelineBuilder: () => {
+                let pipeline = contextPipelineBuilder().compose({ action: this.prepareNavigation.bind(this) });
+
+                for (const hook of this.preNavigationHooks) {
+                    pipeline = pipeline.compose(skipGuard(hook));
+                }
+
+                pipeline = pipeline.compose(skipGuard(this.navigate.bind(this)));
+
+                for (const hook of this.postNavigationHooks) {
+                    pipeline = pipeline.compose(skipGuard(hook));
+                }
+
+                return pipeline
+                    .compose(skipGuard(this.finalizeNavigation.bind(this)))
                     .compose({ action: this.handleBlockedRequestByContent.bind(this) })
-                    .compose({ action: this.restoreRequestState.bind(this) }),
+                    .compose({ action: this.restoreRequestState.bind(this) });
+            },
             extendContext: extendContext as (context: Context) => Awaitable<ContextExtension>,
         });
 
@@ -490,6 +527,9 @@ export abstract class BrowserCrawler<
                     "The `response` property is not available. This might mean that you're trying to access it before navigation or that navigation resulted in `null` (this should only happen with `about:` URLs)",
                 );
             },
+            get gotoOptions(): Dictionary {
+                throw new Error('The `gotoOptions` property is not available until `prepareNavigation` runs.');
+            },
             browserController: browserControllerInstance,
             enqueueLinks: async (enqueueOptions: EnqueueLinksOptions = {}) => {
                 return (await browserCrawlerEnqueueLinks({
@@ -506,10 +546,7 @@ export abstract class BrowserCrawler<
         };
     }
 
-    private async performNavigation(crawlingContext: Context): Promise<{
-        request: LoadedRequest<Request>;
-        response?: Response;
-    }> {
+    private async prepareNavigation(crawlingContext: Context): Promise<Partial<Context>> {
         if (crawlingContext.request.skipNavigation) {
             return {
                 request: new Proxy(crawlingContext.request, {
@@ -527,42 +564,56 @@ export abstract class BrowserCrawler<
                         'The `response` property is not available - `skipNavigation` was used',
                     );
                 },
-            };
+            } as Partial<Context>;
         }
 
-        const gotoOptions = { timeout: this.navigationTimeoutMillis } as unknown as GoToOptions;
+        crawlingContext.request.state = RequestState.BEFORE_NAV;
 
-        const preNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
+        return {
+            gotoOptions: { timeout: this.navigationTimeoutMillis } as unknown as GoToOptions,
+            [COOKIES_BEFORE_HOOKS]: this._getCookieHeaderFromRequest(crawlingContext.request),
+        } as unknown as Partial<Context>;
+    }
 
-        crawlingContext.request.state = RequestState.BEFORE_NAV;
-        await this._executeHooks(this.preNavigationHooks, crawlingContext, gotoOptions);
+    private async navigate(crawlingContext: Context): Promise<Partial<Context>> {
         tryCancel();
 
-        const postNavigationHooksCookies = this._getCookieHeaderFromRequest(crawlingContext.request);
+        const gotoOptions = crawlingContext.gotoOptions as GoToOptions;
+        const cookiesBeforeHooks = readContextField<string>(crawlingContext, COOKIES_BEFORE_HOOKS);
+        const cookiesAfterHooks = this._getCookieHeaderFromRequest(crawlingContext.request);
 
-        await this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
+        await this._applyCookies(crawlingContext, cookiesBeforeHooks, cookiesAfterHooks);
 
         let response: Response | undefined;
-
         try {
             response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
         } catch (error) {
             await this._handleNavigationTimeout(crawlingContext, error as Error);
-
             crawlingContext.request.state = RequestState.ERROR;
-
             this._throwIfProxyError(error as Error);
             throw error;
         }
         tryCancel();
 
         crawlingContext.request.state = RequestState.AFTER_NAV;
-        await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
+
+        return { response } as Partial<Context>;
+    }
+
+    private async finalizeNavigation(crawlingContext: Context): Promise<Partial<Context>> {
+        tryCancel();
+
+        let response: Response | undefined;
+        try {
+            response = crawlingContext.response;
+        } catch {
+            // `preparePage` installs a throwing getter for `response`; reaching this branch means
+            // navigation produced no response and no hook overrode it. Treat as undefined.
+        }
 
         await this.processResponse(response, crawlingContext);
         tryCancel();
 
-        // save cookies
         // TODO: Should we save the cookies also after/only the handle page?
         if (this.saveResponseCookies && crawlingContext.session) {
             const cookies = await crawlingContext.browserController.getCookies(crawlingContext.page);
@@ -579,16 +630,7 @@ export abstract class BrowserCrawler<
             }
         }
 
-        if (response !== undefined) {
-            return {
-                request: crawlingContext.request as LoadedRequest<Request>,
-                response,
-            };
-        }
-
-        return {
-            request: crawlingContext.request as LoadedRequest<Request>,
-        };
+        return { request: crawlingContext.request as LoadedRequest<Request> } as Partial<Context>;
     }
 
     private async handleBlockedRequestByContent(