feat: store browser / HTTP client fingerprint in Session

packages/browser-pool/src/fingerprinting/hooks.ts

@@ -1,3 +1,4 @@
+import type { ISession, SessionFingerprint } from '@crawlee/types';
 import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';
 import type { FingerprintInjector } from 'fingerprint-injector';
 
@@ -6,8 +7,22 @@ import type { BrowserPool } from '../browser-pool.js';
 import type { LaunchContext } from '../launch-context.js';
 import { PlaywrightPlugin } from '../playwright/playwright-plugin.js';
 import { PuppeteerPlugin } from '../puppeteer/puppeteer-plugin.js';
+import type { FingerprintGeneratorOptions } from './types.js';
 import { getGeneratorDefaultOptions } from './utils.js';
 
+function applySessionHints(
+    base: FingerprintGeneratorOptions,
+    fingerprint?: SessionFingerprint,
+): FingerprintGeneratorOptions {
+    if (!fingerprint) return base;
+    return {
+        ...base,
+        ...(fingerprint.browser ? { browsers: [{ name: fingerprint.browser }] } : {}),
+        ...(fingerprint.platform ? { operatingSystems: [fingerprint.platform] } : {}),
+        ...(fingerprint.device ? { devices: [fingerprint.device] } : {}),
+    };
+}
+
 /**
  * @internal
  */
@@ -20,21 +35,19 @@ export function createFingerprintPreLaunchHook(browserPool: BrowserPool<any, any
 
     return (_pageId: string, launchContext: LaunchContext) => {
         const { useIncognitoPages } = launchContext;
-        const cacheKey = (launchContext.session as { id: string } | undefined)?.id ?? launchContext.proxyUrl;
+        const session = launchContext.session as ISession | undefined;
+        const cacheKey = session?.id ?? launchContext.proxyUrl;
         const { launchOptions }: { launchOptions: any } = launchContext;
 
-        // If no options are passed we try to pass best default options as possible to match browser and OS.
-        const fingerprintGeneratorFinalOptions =
-            fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
         let fingerprint: BrowserFingerprintWithHeaders;
 
         if (cacheKey && fingerprintCache?.has(cacheKey)) {
             fingerprint = fingerprintCache.get(cacheKey)!;
-        } else if (cacheKey) {
-            fingerprint = fingerprintGenerator!.getFingerprint(fingerprintGeneratorFinalOptions);
-            fingerprintCache?.set(cacheKey, fingerprint);
         } else {
-            fingerprint = fingerprintGenerator!.getFingerprint(fingerprintGeneratorFinalOptions);
+            const baseOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
+            const finalOptions = applySessionHints(baseOptions, session?.fingerprint);
+            fingerprint = fingerprintGenerator!.getFingerprint(finalOptions);
+            if (cacheKey) fingerprintCache?.set(cacheKey, fingerprint);
         }
 
         launchContext.extend({ fingerprint });

packages/core/src/session_pool/fingerprint.ts

@@ -0,0 +1,33 @@
+import type { SessionFingerprint } from '@crawlee/types';
+
+/**
+ * (browser, platform, device) combinations that correspond to setups people
+ * actually run. Anything not listed here (e.g. `edge` on android, `safari` on
+ * windows, `desktop` mobile platforms) is left out so a randomized default
+ * never produces a fingerprint that would itself be a giveaway.
+ */
+const PROFILES_BY_PLATFORM = [
+    { browser: 'chrome', platform: 'windows', device: 'desktop' },
+    { browser: 'firefox', platform: 'windows', device: 'desktop' },
+    { browser: 'edge', platform: 'windows', device: 'desktop' },
+    { browser: 'chrome', platform: 'macos', device: 'desktop' },
+    { browser: 'firefox', platform: 'macos', device: 'desktop' },
+    { browser: 'safari', platform: 'macos', device: 'desktop' },
+    { browser: 'edge', platform: 'macos', device: 'desktop' },
+    { browser: 'chrome', platform: 'linux', device: 'desktop' },
+    { browser: 'firefox', platform: 'linux', device: 'desktop' },
+    { browser: 'chrome', platform: 'android', device: 'mobile' },
+    { browser: 'firefox', platform: 'android', device: 'mobile' },
+    { browser: 'safari', platform: 'ios', device: 'mobile' },
+] as const;
+
+/**
+ * Build a {@apilink SessionFingerprint} whose `platform` matches the host OS
+ * and whose `browser`/`device` are randomized within the realistic profiles for
+ * that platform. Used by {@apilink SessionPool} as the default fingerprint for
+ * freshly created sessions; callers can override by passing their own
+ * `fingerprint` in `sessionOptions`.
+ */
+export function createDefaultSessionFingerprint(): SessionFingerprint {
+    return { ...PROFILES_BY_PLATFORM[Math.floor(Math.random() * PROFILES_BY_PLATFORM.length)] };
+}

packages/core/src/session_pool/session.ts

@@ -1,4 +1,4 @@
-import type { Dictionary, ISession, ProxyInfo, SessionState } from '@crawlee/types';
+import type { Dictionary, ISession, ProxyInfo, SessionFingerprint, SessionState } from '@crawlee/types';
 import ow from 'ow';
 import { CookieJar } from 'tough-cookie';
 
@@ -68,6 +68,13 @@ export interface SessionOptions {
     errorScore?: number;
     cookieJar?: CookieJar;
     proxyInfo?: ProxyInfo;
+
+    /**
+     * Browser / HTTP client fingerprint tied to this session. Backends use this to make
+     * repeated requests with the same session look consistent (same user-agent, headers,
+     * TLS profile). See {@apilink SessionFingerprint}.
+     */
+    fingerprint?: SessionFingerprint;
 }
 
 /**
@@ -89,6 +96,7 @@ export class Session implements ISession {
     private _retired = false;
     private _proxyInfo?: ProxyInfo;
     private _cookieJar: CookieJar;
+    private _fingerprint?: SessionFingerprint;
     private log: CrawleeLogger;
 
     get errorScore() {
@@ -127,6 +135,14 @@ export class Session implements ISession {
         return this._proxyInfo;
     }
 
+    get fingerprint(): SessionFingerprint | undefined {
+        return this._fingerprint;
+    }
+
+    set fingerprint(fingerprint: SessionFingerprint | undefined) {
+        this._fingerprint = fingerprint;
+    }
+
     /**
      * `true` once {@apilink Session.retire|`retire()`} has been called. Retirement is terminal:
      * a retired session is never picked by the pool and cannot be revived via `markGood()`.
@@ -156,6 +172,7 @@ export class Session implements ISession {
                 maxUsageCount: ow.optional.number,
                 retired: ow.optional.boolean,
                 log: ow.optional.object,
+                fingerprint: ow.optional.object,
             }),
         );
 
@@ -173,6 +190,7 @@ export class Session implements ISession {
             maxUsageCount = 50,
             retired = false,
             log = serviceLocator.getLogger(),
+            fingerprint,
         } = options;
 
         const { expiresAt = getDefaultCookieExpirationDate(maxAgeSecs) } = options;
@@ -181,6 +199,7 @@ export class Session implements ISession {
 
         this._cookieJar = (cookieJar.setCookie as unknown) ? cookieJar : CookieJar.fromJSON(JSON.stringify(cookieJar));
         this._proxyInfo = proxyInfo;
+        this._fingerprint = fingerprint;
         this.id = id;
         this.userData = userData;
         this._maxErrorScore = maxErrorScore;
@@ -252,6 +271,7 @@ export class Session implements ISession {
             cookieJar: this.cookieJar.toJSON()!,
             proxyInfo: this._proxyInfo,
             userData: this.userData,
+            fingerprint: this._fingerprint,
             maxErrorScore: this.maxErrorScore,
             errorScoreDecrement: this.errorScoreDecrement,
             expiresAt: this.expiresAt.toISOString(),

packages/core/src/session_pool/session_pool.ts

@@ -9,6 +9,7 @@ import type { CrawleeLogger } from '../log.js';
 import { serviceLocator } from '../service_locator.js';
 import { KeyValueStore } from '../storages/key_value_store.js';
 import { MAX_POOL_SIZE, PERSIST_STATE_KEY } from './consts.js';
+import { createDefaultSessionFingerprint } from './fingerprint.js';
 import type { SessionOptions } from './session.js';
 import { Session } from './session.js';
 
@@ -437,9 +438,19 @@ export class SessionPool implements ISessionPool {
     /**
      * Invokes `createSessionFunction` with `sessionOptions` already merged from pool-wide defaults and
      * the supplied per-call overrides, so custom implementations don't need to spread `pool.sessionOptions` themselves.
+     *
+     * A default {@apilink SessionFingerprint} is generated up front (host OS as
+     * `platform`, a random valid `browser`/`device` for that platform). Pool-wide
+     * and per-call options override it, and a persisted fingerprint coming
+     * through `_maybeLoadSessionPool` naturally wins because it arrives in
+     * `perCallOptions`.
      */
     private async _invokeCreateSessionFunction(perCallOptions?: SessionOptions): Promise<Session> {
-        const sessionOptions = { ...this.sessionOptions, ...perCallOptions };
+        const sessionOptions: SessionOptions = {
+            fingerprint: createDefaultSessionFingerprint(),
+            ...this.sessionOptions,
+            ...perCallOptions,
+        };
         return this.createSessionFunction({ sessionOptions });
     }
 

packages/http-client/src/base-http-client.ts

@@ -1,8 +1,38 @@
-import type { BaseHttpClient as BaseHttpClientInterface, CrawleeLogger, SendRequestOptions } from '@crawlee/types';
+import type {
+    BaseHttpClient as BaseHttpClientInterface,
+    CrawleeLogger,
+    SendRequestOptions,
+    SessionFingerprint,
+} from '@crawlee/types';
 import { CookieJar } from 'tough-cookie';
 
+/**
+ * Per-request options handed to a concrete client's `fetch` implementation.
+ */
 export interface CustomFetchOptions {
+    /**
+     * Effective proxy URL for this request. `sendRequest` populates this from
+     * the explicit `SendRequestOptions.proxyUrl` override when set, falling back
+     * to `session.proxyInfo.url`.
+     */
     proxyUrl?: string;
+
+    /**
+     * Effective cookie jar for this request. `sendRequest` populates this from
+     * the explicit `SendRequestOptions.cookieJar` override when set, falling
+     * back to `session.cookieJar` (or a fresh jar when neither is provided).
+     */
+    cookieJar?: CookieJar;
+
+    /**
+     * Hints about which browser-like setup this request should be impersonating —
+     * `browser`, `platform`, `device`, and an opaque `details` slot for richer
+     * payloads (e.g. a full browser fingerprint from `fingerprint-generator`).
+     * These are *suggestions*, not requirements: each client applies what it can
+     * (e.g. impit maps `browser` to its impersonation profile) and ignores the
+     * rest on a best-effort basis. Sourced from `SendRequestOptions.session.fingerprint`.
+     */
+    fingerprint?: SessionFingerprint;
 }
 
 /**
@@ -74,11 +104,17 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface {
         proxyUrl?: string;
         cookieJar: CookieJar;
         signal?: AbortSignal;
+        fingerprint?: SessionFingerprint;
     } {
         const proxyUrl = options?.proxyUrl ?? options?.session?.proxyInfo?.url;
         const cookieJar = options?.cookieJar ?? options?.session?.cookieJar ?? new CookieJar();
         const signal = this.createAbortSignal(options?.signal, options?.timeoutMillis);
-        return { proxyUrl, cookieJar: cookieJar as CookieJar, signal };
+        return {
+            proxyUrl,
+            cookieJar: cookieJar as CookieJar,
+            signal,
+            fingerprint: options?.session?.fingerprint,
+        };
     }
 
     private createAbortSignal(signal?: AbortSignal, timeoutMillis?: number): AbortSignal | undefined {
@@ -135,7 +171,7 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface {
         let currentRequest = initialRequest;
         let redirectCount = 0;
 
-        const { proxyUrl, cookieJar, signal } = this.resolveRequestContext(options);
+        const { proxyUrl, cookieJar, signal, fingerprint } = this.resolveRequestContext(options);
         currentRequest = initialRequest.clone();
 
         while (true) {
@@ -144,6 +180,8 @@ export abstract class BaseHttpClient implements BaseHttpClientInterface {
             const response = await this.fetch(currentRequest, {
                 signal,
                 proxyUrl,
+                cookieJar,
+                fingerprint,
                 redirect: 'manual',
             });
 

packages/impit-client/package.json

@@ -50,7 +50,7 @@
         "@apify/datastructures": "^2.0.3",
         "@crawlee/http-client": "workspace:*",
         "@crawlee/types": "workspace:*",
-        "impit": "^0.9.0",
+        "impit": "^0.14.1",
         "tough-cookie": "^6.0.0"
     }
 }

packages/impit-client/src/index.ts

@@ -1,11 +1,32 @@
 import type { CustomFetchOptions } from '@crawlee/http-client';
 import { BaseHttpClient, ResponseWithUrl } from '@crawlee/http-client';
-import type { CrawleeLogger } from '@crawlee/types';
-import { Impit, type ImpitOptions } from 'impit';
+import type { CrawleeLogger, SessionFingerprint } from '@crawlee/types';
+import { Impit, type Browser as ImpitBrowser, type ImpitOptions } from 'impit';
 import type { CookieJar as ToughCookieJar } from 'tough-cookie';
 
 import { LruCache } from '@apify/datastructures';
 
+// Concrete impit impersonation profiles per browser family. The plain `chrome` /
+// `firefox` aliases fall back to the oldest available version, which is a
+// fingerprint giveaway — we pick one of these explicitly instead. Keep in sync
+// with impit's `Browser` type when bumping the dependency.
+const IMPIT_VERSIONS_BY_BROWSER: Partial<Record<NonNullable<SessionFingerprint['browser']>, ImpitBrowser[]>> = {
+    chrome: [
+        'chrome100',
+        'chrome101',
+        'chrome104',
+        'chrome107',
+        'chrome110',
+        'chrome116',
+        'chrome124',
+        'chrome125',
+        'chrome131',
+        'chrome136',
+        'chrome142',
+    ],
+    firefox: ['firefox128', 'firefox133', 'firefox135', 'firefox144'],
+};
+
 export const Browser = {
     'Chrome': 'chrome',
     'Firefox': 'firefox',
@@ -25,6 +46,13 @@ export class ImpitHttpClient extends BaseHttpClient {
      */
     private clientCache: LruCache<{ client: Impit; cookieJar: ToughCookieJar }> = new LruCache({ maxLength: 10 });
 
+    /**
+     * Stable impit impersonation version per fingerprint object, so the same
+     * session keeps impersonating the same browser version across requests
+     * instead of rerolling on every call.
+     */
+    private impitBrowserByFingerprint = new WeakMap<SessionFingerprint, ImpitBrowser>();
+
     private getClient(options: ImpitOptions) {
         const { cookieJar, ...rest } = options;
 
@@ -50,10 +78,13 @@ export class ImpitHttpClient extends BaseHttpClient {
      * @inheritDoc
      */
     async fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response> {
-        const { proxyUrl, redirect, signal } = options ?? {};
+        const { proxyUrl, redirect, signal, fingerprint } = options ?? {};
+
+        const impitBrowser = this.resolveImpitBrowser(fingerprint);
 
         const impit = this.getClient({
             ...this.impitOptions,
+            ...(impitBrowser ? { browser: impitBrowser } : {}),
             proxyUrl,
             followRedirects: redirect === 'follow',
         });
@@ -63,4 +94,17 @@ export class ImpitHttpClient extends BaseHttpClient {
         // todo - cast shouldn't be needed here, impit returns `Uint8Array`
         return new ResponseWithUrl(response.body, response);
     }
+
+    private resolveImpitBrowser(fingerprint?: SessionFingerprint): ImpitBrowser | undefined {
+        if (!fingerprint?.browser) return undefined;
+        const versions = IMPIT_VERSIONS_BY_BROWSER[fingerprint.browser];
+        if (!versions?.length) return undefined;
+
+        const cached = this.impitBrowserByFingerprint.get(fingerprint);
+        if (cached) return cached;
+
+        const picked = versions[Math.floor(Math.random() * versions.length)];
+        this.impitBrowserByFingerprint.set(fingerprint, picked);
+        return picked;
+    }
 }