feat: export SessionPool interface, minimize ISession, allow custom implementations (#3644)

Exports `ISessionPool` interface that can be used to implement custom
`SessionPool` implementations. Adds documentation.

Author: barjin

docs/guides/session_management_basic.ts

@@ -17,7 +17,7 @@ const crawler = new BasicCrawler({
             headers: {
                 // If you want to use the cookieJar.
                 // This way you get the Cookie headers string from session.
-                Cookie: session?.getCookieString(url) ?? '',
+                Cookie: session?.cookieJar.getCookieStringSync(url) ?? '',
             },
         });
         let response;

docs/guides/session_management_standalone.ts

@@ -11,10 +11,10 @@ const sessionPool = new SessionPool(sessionPoolOptions);
 const session = await sessionPool.getSession();
 
 // Increase the errorScore.
-session.markBad();
+session?.markBad();
 
 // Throw away the session.
-session.retire();
+session?.retire();
 
 // Lower the errorScore and mark the session good.
-session.markGood();
+session?.markGood();

docs/upgrading/upgrading_v4.md

@@ -136,10 +136,63 @@ new SessionPool({
 
 ## `Session` no longer requires a `sessionPool` reference
 
-`Session` no longer holds a back-reference to its `SessionPool` and no longer emits a `sessionRetired` event when retired. The `sessionPool` constructor option is gone, `SessionPool` is no longer an `EventEmitter`, and the `EVENT_SESSION_RETIRED` constant is no longer exported.
+`Session` no longer holds a back-reference to its `SessionPool` and no longer emits a `sessionRetired` event when retired. The `sessionPool` constructor option is gone, `SessionPool` is no longer an `EventEmitter`, and the `EVENT_SESSION_RETIRED` constant is no longer exported. Custom `createSessionFunction` implementations that constructed `Session` instances manually should drop the `sessionPool` argument.
+
+**Before:**
+```typescript
+new SessionPool({
+    createSessionFunction: async (pool, opts) =>
+        new Session({ ...opts?.sessionOptions, sessionPool: pool }),
+});
+```
+
+**After:**
+```typescript
+new SessionPool({
+    createSessionFunction: async (opts) =>
+        new Session({ ...opts?.sessionOptions }),
+});
+```
 
 If you previously subscribed to `sessionRetired` on the pool to clean up resources tied to a session, perform the cleanup at the end of your request handler (or via a context-pipeline cleanup hook) by checking `session.isUsable()` instead. `Session.retire()` is now a terminal state — once retired, `isUsable()` returns `false` permanently and cannot be undone by a subsequent `markGood()`.
 
+## Custom `SessionPool` implementations via the `ISessionPool` interface
+
+Crawlers now accept any object implementing the new `ISessionPool` interface as their `sessionPool` option, not just instances of the built-in `SessionPool`. The contract is intentionally tiny — a single method, `getSession()` / `getSession(id)`, that hands out a `Session` for a request. Lifecycle (reset, teardown) is the responsibility of whoever owns the pool: a custom pool you construct yourself is never owned by the crawler, so the crawler never tears it down. This makes it straightforward to plug in a remote, shared, or otherwise customized session-management strategy without subclassing `SessionPool` or copying its internals.
+
+```typescript
+import { BasicCrawler, Session, type ISessionPool } from '@crawlee/core';
+
+class MySessionPool implements ISessionPool {
+    private readonly sessions = new Map<string, Session>();
+
+    async getSession(): Promise<Session>;
+    async getSession(sessionId: string): Promise<Session | undefined>;
+    async getSession(sessionId?: string): Promise<Session | undefined> {
+        if (sessionId) {
+            const existing = this.sessions.get(sessionId);
+            return existing?.isUsable() ? existing : undefined;
+        }
+
+        const usable = [...this.sessions.values()].find((s) => s.isUsable());
+        if (usable) return usable;
+
+        const fresh = new Session();
+        this.sessions.set(fresh.id, fresh);
+        return fresh;
+    }
+}
+
+const crawler = new BasicCrawler({
+    sessionPool: new MySessionPool(),
+    requestHandler: async ({ session }) => {
+        // session is a Session instance, use it as usual
+    },
+});
+```
+
+The returned objects must be `Session` instances — the rest of the crawler relies on `session.markGood()`, `session.cookieJar`, `session.proxyInfo`, and the rest of the concrete `Session` API.
+
 ## `retireOnBlockedStatusCodes` is removed from `Session`
 
 `Session.retireOnBlockedStatusCodes` is removed. Blocked status code handling is now internal to the crawler. Configure blocked status codes via the `blockedStatusCodes` crawler option (moved from `sessionPoolOptions`).

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -70,6 +70,8 @@ import type {
     BaseHttpClient,
     BatchAddRequestsResult,
     Dictionary,
+    ISession,
+    ISessionPool,
     ProxyInfo,
     SetStatusMessageOptions,
     StorageClient,
@@ -302,11 +304,14 @@ export interface BasicCrawlerOptions<
     keepAlive?: boolean;
 
     /**
-     * An existing {@apilink SessionPool} instance to use. When provided, the crawler will use this
-     * pool directly instead of creating a new one, enabling session sharing across multiple crawlers.
-     * The crawler will not tear down a shared pool — the caller is responsible for its lifecycle.
+     * An existing session pool instance to use. When provided, the crawler will use this pool directly instead of
+     * creating a new one, enabling session sharing across multiple crawlers. The crawler will not tear down a shared
+     * pool — the caller is responsible for its lifecycle.
+     *
+     * Accepts the built-in {@apilink SessionPool} or any object implementing the {@apilink ISessionPool} interface,
+     * so custom session-management strategies can be plugged in.
      */
-    sessionPool?: SessionPool;
+    sessionPool?: ISessionPool;
 
     /**
      * Defines the length of the interval for calling the `setStatusMessage` in seconds.
@@ -556,14 +561,18 @@ export class BasicCrawler<
     protected requestManager?: IRequestManager;
 
     /**
-     * A reference to the underlying {@apilink SessionPool} class that manages the crawler's {@apilink Session|sessions}.
+     * A reference to the underlying session pool that manages the crawler's {@apilink Session|sessions}. Typed as
+     * {@apilink ISessionPool} so custom implementations can be plugged in via the `sessionPool` constructor option.
      */
-    sessionPool: SessionPool;
+    sessionPool: ISessionPool;
 
     /**
-     * Indicates whether the crawler owns the session pool (it was not passed from the outside using the `sessionPool` constructor option).
+     * Set when the crawler constructed its own {@apilink SessionPool} (no `sessionPool` option was provided).
+     * Holds the same instance as `sessionPool`, but typed as the concrete class so the crawler can call
+     * lifecycle methods (`resetStore`, `teardown`) that aren't part of {@apilink ISessionPool}. A user-supplied
+     * pool is never owned and never torn down by the crawler.
      */
-    private ownsSessionPool: boolean;
+    private ownedSessionPool?: SessionPool;
 
     /**
      * A reference to the underlying {@apilink AutoscaledPool} class that manages the concurrency of the crawler.
@@ -675,7 +684,7 @@ export class BasicCrawler<
         maxRequestsPerCrawl: ow.optional.number,
         maxCrawlDepth: ow.optional.number,
         autoscaledPoolOptions: ow.optional.object,
-        sessionPool: ow.optional.object.instanceOf(SessionPool),
+        sessionPool: ow.optional.object.validate(validators.sessionPool),
         proxyConfiguration: ow.optional.object.validate(validators.proxyConfiguration),
 
         statusMessageLoggingInterval: ow.optional.number,
@@ -866,18 +875,19 @@ export class BasicCrawler<
                 );
             }
 
-            this.sessionPool =
-                sessionPool ??
-                new SessionPool({
+            if (sessionPool) {
+                this.sessionPool = sessionPool;
+            } else {
+                this.ownedSessionPool = new SessionPool({
                     createSessionFunction: async (opts) =>
                         new Session({
                             ...opts?.sessionOptions,
                             proxyInfo:
                                 opts?.sessionOptions?.proxyInfo ?? (await this.proxyConfiguration?.newProxyInfo()),
                         }),
                 });
-
-            this.ownsSessionPool = !sessionPool;
+                this.sessionPool = this.ownedSessionPool;
+            }
 
             this.blockedStatusCodes = new Set(blockedStatusCodesInput ?? BLOCKED_STATUS_CODES);
 
@@ -1111,26 +1121,22 @@ export class BasicCrawler<
     private async resolveSession({ request }: { request: Request }) {
         const session = await this._timeoutAndRetry(
             async () => {
-                if (request.sessionId) {
-                    const existingSession = await this.sessionPool!.getSession(request.sessionId);
+                const existingSession = await this.sessionPool.getSession(request.sessionId);
 
-                    if (!existingSession) {
-                        throw new ContextPipelineInitializationError(new MissingSessionError(request.sessionId));
-                    }
-
-                    return existingSession;
+                if (!existingSession) {
+                    throw new ContextPipelineInitializationError(new MissingSessionError(request.sessionId));
                 }
 
-                return await this.sessionPool!.getSession();
+                return existingSession;
             },
             this.internalTimeoutMillis,
             `Fetching session timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,
         );
 
-        return { session, proxyInfo: session.proxyInfo };
+        return { session, proxyInfo: session?.proxyInfo };
     }
 
-    private async createContextHelpers({ request, session }: { request: Request; session: Session }) {
+    private async createContextHelpers({ request, session }: { request: Request; session: ISession }) {
         const enqueueLinksWrapper: CrawlingContext['enqueueLinks'] = async (options) => {
             const requestQueue = await this.getRequestQueue();
 
@@ -1296,9 +1302,7 @@ export class BasicCrawler<
 
             this.stats.reset();
             await this.stats.resetStore();
-            if (this.ownsSessionPool) {
-                await this.sessionPool.resetStore();
-            }
+            await this.ownedSessionPool?.resetStore();
         }
 
         this.unexpectedStop = false;
@@ -2241,9 +2245,7 @@ export class BasicCrawler<
             await serviceLocator.getEventManager().close();
         }
 
-        if (this.ownsSessionPool) {
-            await this.sessionPool.teardown();
-        }
+        await this.ownedSessionPool?.teardown();
 
         await this.autoscaledPool?.abort();
     }
@@ -2337,7 +2339,7 @@ export class BasicCrawler<
 
 export interface CreateContextOptions {
     request: Request;
-    session: Session;
+    session: ISession;
     proxyInfo?: ProxyInfo;
 }
 

packages/basic-crawler/src/internals/send-request.ts

@@ -1,5 +1,5 @@
-import type { Request as CrawleeRequest, Session } from '@crawlee/core';
-import type { BaseHttpClient, HttpRequestOptions, SendRequestOptions } from '@crawlee/types';
+import type { Request as CrawleeRequest } from '@crawlee/core';
+import type { BaseHttpClient, HttpRequestOptions, ISession, SendRequestOptions } from '@crawlee/types';
 
 /**
  * Prepares a function to be used as the `sendRequest` context helper.
@@ -9,7 +9,7 @@ import type { BaseHttpClient, HttpRequestOptions, SendRequestOptions } from '@cr
  * @param originRequest The crawling request being processed.
  * @param session The user session associated with the current request.
  */
-export function createSendRequest(httpClient: BaseHttpClient, originRequest: CrawleeRequest, session: Session) {
+export function createSendRequest(httpClient: BaseHttpClient, originRequest: CrawleeRequest, session: ISession) {
     return async (
         overrideRequest: Partial<HttpRequestOptions> = {},
         overrideOptions: SendRequestOptions = {},

packages/basic-crawler/test/batch-add-requests.test.ts

@@ -1,6 +1,7 @@
 import { BasicCrawler } from '@crawlee/basic';
 
 import { MemoryStorageEmulator } from '../../../test/shared/MemoryStorageEmulator.js';
+import { SessionPool } from '@crawlee/core';
 
 describe('BasicCrawler#addRequests with big batch sizes', () => {
     const localStorageEmulator = new MemoryStorageEmulator();
@@ -74,17 +75,20 @@ describe('BasicCrawler - request.sessionId', () => {
     });
 
     test('uses the session matching request.sessionId from the session pool', async () => {
+        const REQUESTED_SESSION_ID = 'my-session';
         let resolvedSessionId: string | undefined;
 
+        const sessionPool = new SessionPool();
+        sessionPool.addSession({ id: REQUESTED_SESSION_ID });
+
         const crawler = new BasicCrawler({
             requestHandler({ session }) {
                 resolvedSessionId = session.id;
             },
+            sessionPool,
         });
 
-        const session = await crawler.sessionPool.newSession({ id: 'my-session' });
-
-        await crawler.run([{ url: 'http://localhost', sessionId: session.id }]);
+        await crawler.run([{ url: 'http://localhost', sessionId: REQUESTED_SESSION_ID }]);
 
         expect(resolvedSessionId).toBe('my-session');
     });
@@ -104,7 +108,7 @@ describe('BasicCrawler - request.sessionId', () => {
 
         expect(errors).toHaveLength(1);
         expect(errors[0].message).toContain(
-            "The current SessionPool instance doesn't contain the requested sessionId: nonexistent",
+            "The current SessionPool instance couldn't find a valid session for the following id: nonexistent",
         );
     });
 });

packages/browser-crawler/src/internals/browser-crawler.ts

@@ -10,11 +10,11 @@ import type {
     Request,
     RequestHandler,
     RequestProvider,
-    Session,
     SkippedRequestCallback,
 } from '@crawlee/basic';
 import {
     BasicCrawler,
+    browserPoolCookieToToughCookie,
     ContextPipeline,
     cookieStringToToughCookie,
     enqueueLinks,
@@ -23,6 +23,7 @@ import {
     RequestState,
     resolveBaseUrlForEnqueueLinksFiltering,
     SessionError,
+    toughCookieToBrowserPoolCookie,
     tryAbsoluteURL,
     validators,
 } from '@crawlee/basic';
@@ -36,7 +37,7 @@ import type {
     LaunchContext,
 } from '@crawlee/browser-pool';
 import { BrowserPool } from '@crawlee/browser-pool';
-import type { BatchAddRequestsResult, Cookie as CookieObject } from '@crawlee/types';
+import type { BatchAddRequestsResult, Cookie as CookieObject, ISession } from '@crawlee/types';
 import type { RobotsTxtFile } from '@crawlee/utils';
 import { CLOUDFLARE_RETRY_CSS_SELECTORS, RETRY_CSS_SELECTORS, sleep } from '@crawlee/utils';
 import ow from 'ow';
@@ -400,7 +401,7 @@ export abstract class BrowserCrawler<
             action: this.preparePage.bind(this),
             cleanup: async (context: {
                 page: Page;
-                session: Session;
+                session: ISession;
                 browserController: ProvidedController;
                 registerDeferredCleanup: BasicCrawlingContext['registerDeferredCleanup'];
             }) => {
@@ -563,10 +564,19 @@ export abstract class BrowserCrawler<
 
         // save cookies
         // TODO: Should we save the cookies also after/only the handle page?
-        if (this.saveResponseCookies) {
+        if (this.saveResponseCookies && crawlingContext.session) {
             const cookies = await crawlingContext.browserController.getCookies(crawlingContext.page);
             tryCancel();
-            crawlingContext.session?.setCookies(cookies, crawlingContext.request.loadedUrl!);
+            const url = crawlingContext.request.loadedUrl!;
+            for (const cookie of cookies) {
+                try {
+                    crawlingContext.session.cookieJar.setCookieSync(browserPoolCookieToToughCookie(cookie), url, {
+                        ignoreError: false,
+                    });
+                } catch (e) {
+                    this.log.debug(`Could not set cookie: ${(e as Error).message}`);
+                }
+            }
         }
 
         if (response !== undefined) {
@@ -602,7 +612,7 @@ export abstract class BrowserCrawler<
         preHooksCookies: string,
         postHooksCookies: string,
     ) {
-        const sessionCookie = session?.getCookies(request.url) ?? [];
+        const sessionCookie = session?.cookieJar.getCookiesSync(request.url).map(toughCookieToBrowserPoolCookie) ?? [];
         const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
         const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
 

packages/core/src/cookie_utils.ts

@@ -59,19 +59,24 @@ export function toughCookieToBrowserPoolCookie(toughCookie: Cookie): CookieObjec
 /**
  * Transforms browser-pool cookie to tough-cookie.
  * @param cookieObject Cookie object (for instance from the `page.cookies` method).
+ * @param maxAgeSecs Fallback expiration in seconds when the cookie itself has no `expires`.
+ * When omitted, such a cookie is stored as a session cookie (no automatic expiration).
  * @internal
  */
-export function browserPoolCookieToToughCookie(cookieObject: CookieObject, maxAgeSecs: number) {
+export function browserPoolCookieToToughCookie(cookieObject: CookieObject, maxAgeSecs?: number) {
     const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;
-    const expires = isExpiresValid
-        ? new Date(cookieObject.expires! * 1000)
-        : getDefaultCookieExpirationDate(maxAgeSecs);
+    let expires: Date | 'Infinity' | undefined;
+    if (isExpiresValid) {
+        expires = new Date(cookieObject.expires! * 1000);
+    } else if (maxAgeSecs != null) {
+        expires = getDefaultCookieExpirationDate(maxAgeSecs);
+    }
     const domainHasLeadingDot = cookieObject.domain?.startsWith?.('.');
     const domain = domainHasLeadingDot ? cookieObject.domain?.slice?.(1) : cookieObject.domain;
     return new Cookie({
         key: cookieObject.name,
         value: cookieObject.value,
-        expires,
+        ...(expires !== undefined && { expires }),
         domain,
         path: cookieObject.path,
         secure: cookieObject.secure,