chore: fix broken types in docs examples (#3287)

barjin · web-flow · commit ab98f68e1c51 · 2025-12-01T16:17:11.000+01:00
Closes #3277
diff --git a/docs/examples/file_download.ts b/docs/examples/file_download.ts
@@ -2,11 +2,11 @@ import { FileDownload } from 'crawlee';
 
 // Create a FileDownload - a custom crawler instance that will download files from URLs.
 const crawler = new FileDownload({
-    async requestHandler({ body, request, contentType, getKeyValueStore }) {
+    async requestHandler({ request, response, contentType, getKeyValueStore }) {
         const url = new URL(request.url);
         const kvs = await getKeyValueStore();
 
-        await kvs.setValue(url.pathname.replace(/\//g, '_'), body, { contentType: contentType.type });
+        await kvs.setValue(url.pathname.replace(/\//g, '_'), response.body, { contentType: contentType.type });
     },
 });
 
diff --git a/docs/examples/file_download_stream.ts b/docs/examples/file_download_stream.ts
@@ -23,32 +23,27 @@ function createProgressTracker({ url, log, totalBytes }: { url: URL; log: Log; t
 
 // Create a FileDownload - a custom crawler instance that will download files from URLs.
 const crawler = new FileDownload({
-    async streamHandler({ stream, request, log, getKeyValueStore }) {
+    async requestHandler({ response, request, log, getKeyValueStore }) {
         const url = new URL(request.url);
 
         log.info(`Downloading ${url} to ${url.pathname.replace(/\//g, '_')}...`);
 
-        await new Promise<void>((resolve, reject) => {
-            // With the 'response' event, we have received the headers of the response.
-            stream.on('response', async (response) => {
-                const kvs = await getKeyValueStore();
-                await kvs.setValue(
-                    url.pathname.replace(/\//g, '_'),
-                    pipeline(
-                        stream,
-                        createProgressTracker({ url, log, totalBytes: Number(response.headers['content-length']) }),
-                        (error) => {
-                            if (error) reject(error);
-                        },
-                    ),
-                    { contentType: response.headers['content-type'] },
-                );
-
-                log.info(`Downloaded ${url} to ${url.pathname.replace(/\//g, '_')}.`);
-
-                resolve();
-            });
-        });
+        if (!response.body) return;
+
+        const kvs = await getKeyValueStore();
+        await kvs.setValue(
+            url.pathname.replace(/\//g, '_'),
+            pipeline(
+                response.body,
+                createProgressTracker({ url, log, totalBytes: Number(response.headers.get('content-length')) }),
+                (error) => {
+                    if (error) log.error(`Failed to download ${url}: ${error.message}`);
+                },
+            ),
+            response.headers.get('content-type') ? { contentType: response.headers.get('content-type')! } : {},
+        );
+
+        log.info(`Downloaded ${url} to ${url.pathname.replace(/\//g, '_')}.`);
     },
 });
 
diff --git a/docs/guides/custom-http-client/implementation.ts b/docs/guides/custom-http-client/implementation.ts
@@ -11,7 +11,7 @@ import { Readable } from 'node:stream';
 export class CustomHttpClient implements BaseHttpClient {
     async sendRequest<TResponseType extends keyof ResponseTypes = 'text'>(
         request: HttpRequest<TResponseType>,
-    ): Promise<HttpResponse<TResponseType>> {
+    ): Promise<Response> {
         const requestHeaders = new Headers();
         for (let [headerName, headerValues] of Object.entries(request.headers ?? {})) {
             if (headerValues === undefined) {
@@ -27,96 +27,20 @@ export class CustomHttpClient implements BaseHttpClient {
             }
         }
 
-        const response = await fetch(request.url, {
+        return fetch(request.url, {
             method: request.method,
             headers: requestHeaders,
-            body: request.body as string, // TODO implement stream/generator handling
+            body: request.body as string,
             signal: request.signal,
-            // TODO implement the rest of request parameters (e.g., timeout, proxyUrl, cookieJar, ...)
         });
-
-        const headers: Record<string, string> = {};
-
-        response.headers.forEach((value, headerName) => {
-            headers[headerName] = value;
-        });
-
-        return {
-            complete: true,
-            request,
-            url: response.url,
-            statusCode: response.status,
-            redirectUrls: [], // TODO you need to handle redirects manually to track them
-            headers,
-            trailers: {}, // TODO not supported by fetch
-            ip: undefined,
-            body:
-                request.responseType === 'text'
-                    ? await response.text()
-                    : request.responseType === 'json'
-                      ? await response.json()
-                      : Buffer.from(await response.text()),
-        };
     }
 
-    async stream(request: HttpRequest, _onRedirect?: RedirectHandler): Promise<StreamingHttpResponse> {
-        const fetchResponse = await fetch(request.url, {
+    async stream(request: HttpRequest, _onRedirect?: RedirectHandler): Promise<Response> {
+        return fetch(request.url, {
             method: request.method,
             headers: new Headers(),
-            body: request.body as string, // TODO implement stream/generator handling
+            body: request.body as string,
             signal: request.signal,
-            // TODO implement the rest of request parameters (e.g., timeout, proxyUrl, cookieJar, ...)
         });
-
-        const headers: Record<string, string> = {}; // TODO same as in sendRequest()
-
-        async function* read() {
-            const reader = fetchResponse.body?.getReader();
-
-            const stream = new ReadableStream({
-                start(controller) {
-                    if (!reader) {
-                        return null;
-                    }
-                    return pump();
-                    function pump(): Promise<void> {
-                        return reader!.read().then(({ done, value }) => {
-                            // When no more data needs to be consumed, close the stream
-                            if (done) {
-                                controller.close();
-                                return;
-                            }
-                            // Enqueue the next data chunk into our target stream
-                            controller.enqueue(value);
-                            return pump();
-                        });
-                    }
-                },
-            });
-
-            for await (const chunk of stream) {
-                yield chunk;
-            }
-        }
-
-        const response = {
-            complete: false,
-            request,
-            url: fetchResponse.url,
-            statusCode: fetchResponse.status,
-            redirectUrls: [], // TODO you need to handle redirects manually to track them
-            headers,
-            trailers: {}, // TODO not supported by fetch
-            ip: undefined,
-            stream: Readable.from(read()),
-            get downloadProgress() {
-                return { percent: 0, transferred: 0 }; // TODO track this
-            },
-            get uploadProgress() {
-                return { percent: 0, transferred: 0 }; // TODO track this
-            },
-        };
-
-        return response;
     }
 }
diff --git a/docs/guides/proxy_management.mdx b/docs/guides/proxy_management.mdx
@@ -31,7 +31,7 @@ import InspectionPuppeteerSource from '!!raw-loader!./proxy_management_inspectio
 and most effective ways of preventing access to a website. It is therefore paramount for
 a good web scraping library to provide easy to use but powerful tools which can work around
 IP blocking. The most powerful weapon in our anti IP blocking arsenal is a
-[proxy server](https://en.wikipedia.org/wiki/Proxy_server). 
+[proxy server](https://en.wikipedia.org/wiki/Proxy_server).
 
 With Crawlee we can use our own proxy servers or proxy servers acquired from
 third-party providers.
@@ -105,7 +105,7 @@ You can also provide a list of proxy tiers to the `ProxyConfiguration` class. Th
 
 :::warning
 
-Note that the `tieredProxyUrls` option requires `ProxyConfiguration` to be used from a crawler instance ([see below](#crawler-integration)). 
+Note that the `tieredProxyUrls` option requires `ProxyConfiguration` to be used from a crawler instance ([see below](#crawler-integration)).
 
 Using this configuration through the `newUrl` calls will not yield the expected results.
 
@@ -162,9 +162,7 @@ Our crawlers will now use the selected proxies for all connections.
 
 ## IP Rotation and session management
 
-&#8203;<ApiLink to="core/class/ProxyConfiguration#newUrl">`proxyConfiguration.newUrl()`</ApiLink> allows us to pass a `sessionId` parameter. It will then be used to create a `sessionId`-`proxyUrl` pair, and subsequent `newUrl()` calls with the same `sessionId` will always return the same `proxyUrl`. This is extremely useful in scraping, because we want to create the impression of a real user. See the [session management guide](../guides/session-management) and <ApiLink to="core/class/SessionPool">`SessionPool`</ApiLink> class for more information on how keeping a real session helps us avoid blocking.
-
-When no `sessionId` is provided, our proxy URLs are rotated round-robin.
+Each call to <ApiLink to="core/class/ProxyConfiguration#newUrl">`proxyConfiguration.newUrl()`</ApiLink> generates a new proxy URL. Crawler instances pair these URLs with `Session` instances and rotate those together with browser fingerprints, impersonated headers, and more. This is extremely useful in scraping, because we want to create the impression of a real user. See the [session management guide](../guides/session-management) and <ApiLink to="core/class/SessionPool">`SessionPool`</ApiLink> class for more information on how keeping a real session helps us avoid blocking.
 
 <Tabs groupId="proxy_session_management">
     <TabItem value="http" label="HttpCrawler">
diff --git a/docs/guides/proxy_management_session_standalone.ts b/docs/guides/proxy_management_session_standalone.ts
@@ -4,10 +4,4 @@ const proxyConfiguration = new ProxyConfiguration({
     /* opts */
 });
 
-const sessionPool = await SessionPool.open({
-    /* opts */
-});
-
-const session = await sessionPool.getSession();
-
-const proxyUrl = await proxyConfiguration.newUrl(session.id);
+const proxyUrl = await proxyConfiguration.newUrl();
diff --git a/docs/guides/session_management_basic.ts b/docs/guides/session_management_basic.ts
@@ -1,5 +1,6 @@
 import { BasicCrawler, ProxyConfiguration } from 'crawlee';
-import { gotScraping } from 'got-scraping';
+import { Impit } from 'impit';
+import { Cookie } from 'tough-cookie';
 
 const proxyConfiguration = new ProxyConfiguration({
     /* opts */
@@ -12,22 +13,19 @@ const crawler = new BasicCrawler({
     sessionPoolOptions: { maxPoolSize: 100 },
     async requestHandler({ request, session }) {
         const { url } = request;
-        const requestOptions = {
-            url,
-            // We use session id in order to have the same proxyUrl
-            // for all the requests using the same session.
-            proxyUrl: await proxyConfiguration.newUrl(session?.id),
-            throwHttpErrors: false,
+        const client = new Impit({
+            proxyUrl: await proxyConfiguration.newUrl(),
+            ignoreTlsErrors: true,
             headers: {
                 // If you want to use the cookieJar.
                 // This way you get the Cookie headers string from session.
-                Cookie: session?.getCookieString(url),
+                Cookie: session?.getCookieString(url) ?? '',
             },
-        };
+        });
         let response;
 
         try {
-            response = await gotScraping(requestOptions);
+            response = await client.fetch(url);
         } catch (e) {
             if (e === 'SomeNetworkError') {
                 // If a network error happens, such as timeout, socket hangup, etc.
@@ -39,9 +37,9 @@ const crawler = new BasicCrawler({
         }
 
         // Automatically retires the session based on response HTTP status code.
-        session?.retireOnBlockedStatusCodes(response.statusCode);
+        session?.retireOnBlockedStatusCodes(response.status);
 
-        if (response.body.includes('You are blocked!')) {
+        if ((await response.text()).includes('You are blocked!')) {
             // You are sure it is blocked.
             // This will throw away the session.
             session?.retire();
@@ -51,6 +49,17 @@ const crawler = new BasicCrawler({
         // No need to call session.markGood -> BasicCrawler calls it for you.
 
         // If you want to use the CookieJar in session you need.
-        session?.setCookiesFromResponse(response);
+        if (response.headers.has('set-cookie')) {
+            const newCookies = response.headers
+                .get('set-cookie')
+                ?.split(';')
+                .map((x) => Cookie.parse(x));
+
+            newCookies?.forEach((cookie) => {
+                if (cookie) {
+                    session?.cookieJar?.setCookie(cookie, url);
+                }
+            });
+        }
     },
 });
diff --git a/docs/package.json b/docs/package.json
@@ -10,6 +10,7 @@
     "typescript": "^5.9.3"
   },
   "dependencies": {
+    "impit": "^0.7.1",
     "playwright-extra": "^4.3.6",
     "puppeteer-extra": "^3.3.6",
     "puppeteer-extra-plugin-stealth": "^2.11.2"
diff --git a/docs/yarn.lock b/docs/yarn.lock
@@ -69,6 +69,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "crawlee-docs@workspace:."
   dependencies:
+    impit: "npm:^0.7.1"
     playwright-extra: "npm:^4.3.6"
     puppeteer-extra: "npm:^3.3.6"
     puppeteer-extra-plugin-stealth: "npm:^2.11.2"
@@ -157,6 +158,95 @@ __metadata:
   languageName: node
   linkType: hard
 
+"impit-darwin-arm64@npm:0.7.1":
+  version: 0.7.1
+  resolution: "impit-darwin-arm64@npm:0.7.1"
+  conditions: os=darwin & cpu=arm64
+  languageName: node
+  linkType: hard
+
+"impit-darwin-x64@npm:0.7.1":
+  version: 0.7.1
+  resolution: "impit-darwin-x64@npm:0.7.1"
+  conditions: os=darwin & cpu=x64
+  languageName: node
+  linkType: hard
+
+"impit-linux-arm64-gnu@npm:0.7.1":
+  version: 0.7.1
+  resolution: "impit-linux-arm64-gnu@npm:0.7.1"
+  conditions: os=linux & cpu=arm64 & libc=glibc
+  languageName: node
+  linkType: hard
+
+"impit-linux-arm64-musl@npm:0.7.1":
+  version: 0.7.1
+  resolution: "impit-linux-arm64-musl@npm:0.7.1"
+  conditions: os=linux & cpu=arm64 & libc=musl
+  languageName: node
+  linkType: hard
+
+"impit-linux-x64-gnu@npm:0.7.1":
+  version: 0.7.1
+  resolution: "impit-linux-x64-gnu@npm:0.7.1"
+  conditions: os=linux & cpu=x64 & libc=glibc
+  languageName: node
+  linkType: hard
+
+"impit-linux-x64-musl@npm:0.7.1":
+  version: 0.7.1
+  resolution: "impit-linux-x64-musl@npm:0.7.1"
+  conditions: os=linux & cpu=x64 & libc=musl
+  languageName: node
+  linkType: hard
+
+"impit-win32-arm64-msvc@npm:0.7.1":
+  version: 0.7.1
+  resolution: "impit-win32-arm64-msvc@npm:0.7.1"
+  conditions: os=win32 & cpu=arm64
+  languageName: node
+  linkType: hard
+
+"impit-win32-x64-msvc@npm:0.7.1":
+  version: 0.7.1
+  resolution: "impit-win32-x64-msvc@npm:0.7.1"
+  conditions: os=win32 & cpu=x64
+  languageName: node
+  linkType: hard
+
+"impit@npm:^0.7.1":
+  version: 0.7.1
+  resolution: "impit@npm:0.7.1"
+  dependencies:
+    impit-darwin-arm64: "npm:0.7.1"
+    impit-darwin-x64: "npm:0.7.1"
+    impit-linux-arm64-gnu: "npm:0.7.1"
+    impit-linux-arm64-musl: "npm:0.7.1"
+    impit-linux-x64-gnu: "npm:0.7.1"
+    impit-linux-x64-musl: "npm:0.7.1"
+    impit-win32-arm64-msvc: "npm:0.7.1"
+    impit-win32-x64-msvc: "npm:0.7.1"
+  dependenciesMeta:
+    impit-darwin-arm64:
+      optional: true
+    impit-darwin-x64:
+      optional: true
+    impit-linux-arm64-gnu:
+      optional: true
+    impit-linux-arm64-musl:
+      optional: true
+    impit-linux-x64-gnu:
+      optional: true
+    impit-linux-x64-musl:
+      optional: true
+    impit-win32-arm64-msvc:
+      optional: true
+    impit-win32-x64-msvc:
+      optional: true
+  checksum: 10c0/25032be7069d725273180c8f7de8c03a8572d786e196ebfaaa93e8fe591f85464ebd4bb766a1de59212ba6aef02e05e547dfcabf11e6e922cb1c58fc722edd2f
+  languageName: node
+  linkType: hard
+
 "inflight@npm:^1.0.4":
   version: 1.0.6
   resolution: "inflight@npm:1.0.6"
