feat: fetch API for custom http clients (#3326)

Removes `HttpClient.stream()`, leaving only `HttpClient.sendRequest()`
(they were both returning a streamable `Response` since
#3295).

Extracts cookie- and redirect-related behaviour to a separate abstract
`BaseHttpClient` class.

Custom HTTP clients now only have to implement one `fetch` method
(matches the native `fetch` API). This makes the custom HTTP client
implementation easier and will hopefully drive community contributions.

Closes #3071 
Closes #3314

Blocked by apify/impit#348

Author: barjin

docs/examples/skip-navigation.ts

@@ -8,10 +8,15 @@ const crawler = new PlaywrightCrawler({
         // The request should have the navigation skipped
         if (request.skipNavigation) {
             // Request the image and get its buffer back
-            const imageResponse = await sendRequest({ responseType: 'buffer' });
-
-            // Save the image in the key-value store
-            await imageStore.setValue(`${request.userData.key}.png`, imageResponse.body);
+            const imageResponse = await sendRequest();
+
+            // Saves the image in the key-value store.
+            //
+            // Note: For large-scale file downloads, consider using FileDownload crawler:
+            // https://crawlee.dev/js/api/http-crawler/class/FileDownload
+            await imageStore.setValue(`${request.userData.key}.svg`, await imageResponse.bytes(), {
+                contentType: 'image/svg+xml',
+            });
 
             // Prevent executing the rest of the code as we do not need it
             return;

docs/guides/custom-http-client/custom-http-client.mdx

@@ -10,14 +10,34 @@ import CodeBlock from '@theme/CodeBlock';
 import ImplementationSource from '!!raw-loader!./implementation.ts';
 import UsageSource from '!!raw-loader!./usage.ts';
 
-The <ApiLink to="basic-crawler/class/BasicCrawler">`BasicCrawler`</ApiLink> class allows you to configure the HTTP client implementation using the `httpClient` constructor option. This might be useful for testing or if you need to swap out the default implementation based on `got-scraping` for something else, such as `curl-impersonate` or `axios`.
+The <ApiLink to="basic-crawler/class/BasicCrawler">`BasicCrawler`</ApiLink> class allows you to configure the HTTP client implementation using the `httpClient` constructor option. This might be useful for testing or if you need to swap out the default implementation based on `got-scraping` for something else, such as `curl-impersonate`.
 
-The HTTP client implementation needs to conform to the <ApiLink to="types/interface/BaseHttpClient">`BaseHttpClient`</ApiLink> interface. For a rough idea on how it might look, see a skeleton implementation that uses the standard `fetch` interface:
+## Built-in HTTP clients
+
+Crawlee provides several HTTP client implementations out of the box:
+
+- **`GotScrapingHttpClient`** (default) - Uses the `got-scraping` library for browser-like requests with support for custom headers, browser fingerprints, and proxies.
+- **`ImpitHttpClient`** - Uses the `impit` library for making requests that closely mimic browser behavior.
+- **`FetchHttpClient`** - Simple implementation using the native `fetch` API (does not support proxies).
+
+## Implementing a custom HTTP client
+
+To create a custom HTTP client, extend the <ApiLink to="http-client/class/BaseHttpClient">`BaseHttpClient`</ApiLink> abstract class from `@crawlee/http-client`. The base class handles common functionality like cookie management, redirect following, session integration, proxy support, and timeout handling.
+
+Your custom implementation only needs to override the `fetch` method to perform the actual network request:
 
 <CodeBlock language="ts">{ImplementationSource}</CodeBlock>
 
+By extending `BaseHttpClient`, your implementation automatically gets:
+- Cookie jar management (applying cookies before requests, saving cookies from responses)
+- Automatic redirect following (up to 10 redirects)
+- Session integration (proxy URL and cookies from session)
+- Timeout handling via AbortSignal
+- Proxy URL support
+
 You may then instantiate it and pass to a crawler constructor:
 
 <CodeBlock language="ts">{UsageSource}</CodeBlock>
 
-Please note that the interface is experimental and it will likely change with Crawlee version 4.
+Alternatively, you can implement the <ApiLink to="types/interface/BaseHttpClient">`BaseHttpClient`</ApiLink> interface directly if you need full control over all aspects of the HTTP request handling, including cookies, redirects, and sessions. However, this approach requires implementing significantly more logic yourself.
+

docs/guides/custom-http-client/implementation.ts

@@ -1,17 +1,14 @@
-import type { BaseHttpClient, SendRequestOptions, StreamOptions } from '@crawlee/types';
+import { BaseHttpClient, type CustomFetchOptions } from '@crawlee/http-client';
 
-export class FetchHttpClient implements BaseHttpClient {
-    async sendRequest(request: Request, options?: SendRequestOptions): Promise<Response> {
-        const signal = options?.timeout ? AbortSignal.timeout(options.timeout ?? 0) : undefined;
-        return fetch(request, {
-            signal,
-        });
-    }
-
-    async stream(request: Request, options: StreamOptions): Promise<Response> {
-        const signal = options?.timeout ? AbortSignal.timeout(options.timeout ?? 0) : undefined;
-        return fetch(request, {
-            signal,
-        });
+/**
+ * A simple HTTP client implementation using the native `fetch` API.
+ *
+ * Custom implementations only need to override the `fetch` method.
+ */
+export class CustomFetchClient extends BaseHttpClient {
+    protected override async fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response> {
+        // The base class handles cookies, redirects, sessions, and timeouts.
+        // We only need to perform the actual network request here.
+        return fetch(request, options);
     }
 }

docs/guides/custom-http-client/usage.ts

@@ -1,8 +1,8 @@
 import { HttpCrawler } from 'crawlee';
-import { FetchHttpClient } from './implementation.js';
+import { CustomFetchClient } from './implementation.js';
 
 const crawler = new HttpCrawler({
-    httpClient: new FetchHttpClient(),
+    httpClient: new CustomFetchClient(),
     async requestHandler() {
         /* ... */
     },

packages/basic-crawler/package.json

@@ -43,11 +43,11 @@
         "@apify/timeout": "^0.3.2",
         "@apify/utilities": "^2.15.5",
         "@crawlee/core": "4.0.0",
+        "@crawlee/got-scraping-client": "4.0.0",
         "@crawlee/types": "4.0.0",
         "@crawlee/utils": "4.0.0",
         "csv-stringify": "^6.5.2",
         "fs-extra": "^11.3.0",
-        "got-scraping": "^4.1.1",
         "ow": "^2.0.0",
         "tldts": "^7.0.6",
         "tslib": "^2.8.1",

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -38,7 +38,6 @@ import {
     enqueueLinks,
     EnqueueStrategy,
     EventType,
-    GotScrapingHttpClient,
     KeyValueStore,
     mergeCookies,
     NonRetryableError,
@@ -57,6 +56,7 @@ import {
     Statistics,
     validators,
 } from '@crawlee/core';
+import { GotScrapingHttpClient } from '@crawlee/got-scraping-client';
 import type {
     Awaitable,
     BaseHttpClient,
@@ -1631,7 +1631,7 @@ export class BasicCrawler<
             },
             pushData: this.pushData.bind(this),
             useState: this.useState.bind(this),
-            sendRequest: createSendRequest(this.httpClient, request!, session) as CrawlingContext['sendRequest'],
+            sendRequest: createSendRequest(this.httpClient, request!, session),
             getKeyValueStore: async (idOrName?: string) => KeyValueStore.open(idOrName, { config: this.config }),
             registerDeferredCleanup: (cleanup) => {
                 deferredCleanup.push(cleanup);

packages/basic-crawler/src/internals/send-request.ts

@@ -18,11 +18,6 @@ export function createSendRequest(
         overrideRequest: Partial<HttpRequestOptions> = {},
         overrideOptions: SendRequestOptions = {},
     ): Promise<Response> => {
-        const sessionCookieJar = {
-            getCookieString: async (url: string) => session?.getCookieString(url),
-            setCookie: async (rawCookie: string, url: string) => session?.setCookie(rawCookie, url),
-        };
-
         const baseRequest = originRequest.intoFetchAPIRequest();
         const mergedUrl = overrideRequest.url ?? baseRequest.url;
         const mergedMethod = overrideRequest.method ?? baseRequest.method;
@@ -42,7 +37,7 @@ export function createSendRequest(
 
         return httpClient.sendRequest(request, {
             session,
-            cookieJar: overrideOptions?.cookieJar ?? (sessionCookieJar as any),
+            cookieJar: overrideOptions?.cookieJar ?? session?.cookieJar,
             timeout: overrideOptions.timeout,
         });
     };

packages/core/package.json

@@ -60,7 +60,6 @@
         "@vladfrangu/async_event_emitter": "^2.4.6",
         "csv-stringify": "^6.5.2",
         "fs-extra": "^11.3.0",
-        "got-scraping": "^4.1.1",
         "json5": "^2.2.3",
         "minimatch": "^10.0.1",
         "ow": "^2.0.0",

packages/core/src/crawlers/crawler_commons.ts

@@ -1,5 +1,4 @@
-import type { Dictionary, ProxyInfo } from '@crawlee/types';
-import type { OptionsInit } from 'got-scraping';
+import type { Dictionary, HttpRequestOptions, ProxyInfo, SendRequestOptions } from '@crawlee/types';
 import type { ReadonlyDeep, SetRequired } from 'type-fest';
 
 import type { Configuration } from '../configuration.js';
@@ -139,8 +138,7 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
     ): Promise<unknown>;
 
     /**
-     * Fires HTTP request via [`got-scraping`](https://crawlee.dev/js/docs/guides/got-scraping), allowing to override the request
-     * options on the fly.
+     * Fires HTTP request via the internal HTTP client, allowing to override the request options on the fly.
      *
      * This is handy when you work with a browser crawler but want to execute some requests outside it (e.g. API requests).
      * Check the [Skipping navigations for certain requests](https://crawlee.dev/js/docs/examples/skip-navigation) example for
@@ -155,7 +153,10 @@ export interface CrawlingContext<UserData extends Dictionary = Dictionary> exten
      * },
      * ```
      */
-    sendRequest(overrideOptions?: Partial<OptionsInit>): Promise<Response>;
+    sendRequest: (
+        requestOverrides?: Partial<HttpRequestOptions>,
+        optionsOverrides?: SendRequestOptions,
+    ) => Promise<Response>;
 
     /**
      * Register a function to be called at the very end of the request handling process. This is useful for resources that should be accessible to error handlers, for instance.