feat: Implement storage aliases (#3636)

closes #3074

Author: janbuchar

docs/upgrading/upgrading_v4.md

@@ -532,6 +532,52 @@ If you implemented a custom `StorageClient`, you need to:
 2. Replace the six getter methods (`dataset`, `datasets`, `keyValueStore`, `keyValueStores`, `requestQueue`, `requestQueues`) with three async factory methods (`createDatasetClient`, `createKeyValueStoreClient`, `createRequestQueueClient`). Each factory should handle both opening an existing storage and creating a new one.
 3. Apply the sub-client renames listed above (`get` → `getMetadata`, `delete` → `drop`, etc.) and implement the new `purge()` method.
 
+## Multiple crawler instances use separate default request queues
+
+In v3, every `BasicCrawler` (or subclass) that didn't receive an explicit `requestQueue` option would open the same default request queue. If you created two crawlers in the same process, they would silently share a queue — leading to request collisions and hard-to-debug deduplication issues.
+
+In v4, only the **first** crawler instance uses the default request queue. Each subsequent instance automatically gets its own queue via an internal alias (e.g. `__default_1__`, `__default_2__`, etc.). This means multiple crawlers can safely coexist without interfering with each other's requests.
+
+If you explicitly pass a `requestQueue` (or `requestManager`) to the crawler, that queue is used as-is regardless of instance order.
+
+## Repeated `run()` calls use `purge()` instead of `drop()` + recreate
+
+When calling `crawler.run()` multiple times on the same crawler instance, v3 would drop the default request queue and create a fresh one between runs. In v4, the crawler **purges** the queue instead — clearing all requests and resetting internal counters, but keeping the same queue object. This is more efficient and avoids edge cases around stale references.
+
+The new `purge()` method is available on `RequestProvider` (the base class for `RequestQueue`) and is also defined as an optional method on the `IRequestManager` interface.
+
+By default, only queues that the crawler created itself (the "owned" queue) are purged between runs — a user-supplied queue is never touched unless you explicitly opt in. The `purgeRequestQueue` option in `CrawlerRunOptions` controls this behavior:
+
+| `purgeRequestQueue` value | Owned queue (auto-created) | User-supplied queue |
+|---|---|---|
+| omitted (default) | Purged | Not purged |
+| `true` | Purged | Purged |
+| `false` | Not purged | Not purged |
+
+```typescript
+// The purge happens automatically between run() calls:
+const crawler = new BasicCrawler({ requestHandler: async ({ request }) => { /* ... */ } });
+await crawler.run(['https://example.com/a', 'https://example.com/b']);
+// Queue is purged here, so the same URLs can be processed again:
+await crawler.run(['https://example.com/a', 'https://example.com/c']);
+```
+
+You can opt out of the automatic purge by passing `purgeRequestQueue: false`:
+
+```typescript
+await crawler.run(urls, { purgeRequestQueue: false });
+```
+
+If you supplied your own `requestQueue` and want it purged between runs, pass `purgeRequestQueue: true` explicitly:
+
+```typescript
+const queue = await RequestQueue.open('my-queue');
+const crawler = new BasicCrawler({ requestQueue: queue, requestHandler: async () => { /* ... */ } });
+await crawler.run(['https://example.com/first']);
+// Explicitly purge the user-supplied queue before the second run:
+await crawler.run(['https://example.com/second'], { purgeRequestQueue: true });
+```
+
 ## Storage `.open()` now also accepts `{ id?, name? }`
 
 `Dataset.open()`, `KeyValueStore.open()`, and `RequestQueue.open()` previously accepted a single `idOrName?: string` parameter. This was ambiguous — callers couldn't express whether they were opening a storage by its ID or by name.

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -537,6 +537,13 @@ export class BasicCrawler<
      */
     private static useStateCrawlerIds = new Set<string>();
 
+    /**
+     * Tracks the number of crawler instances created. The first crawler uses the default
+     * request queue; subsequent ones get their own queue via a unique alias so they don't
+     * collide.
+     */
+    private static instanceCount = 0;
+
     /**
      * A reference to the underlying {@apilink Statistics} class that collects and logs run statistics for requests.
      */
@@ -574,6 +581,13 @@ export class BasicCrawler<
      */
     private ownedSessionPool?: SessionPool;
 
+    /**
+     * Set when the crawler constructed its own {@apilink RequestQueue} (no `requestQueue`/`requestManager`
+     * option was provided). The owned queue is purged (not dropped) between repeated `run()` calls.
+     * A user-supplied queue is never purged by the crawler.
+     */
+    private ownedRequestQueue?: RequestProvider;
+
     /**
      * A reference to the underlying {@apilink AutoscaledPool} class that manages the concurrency of the crawler.
      * > *NOTE:* This property is only initialized after calling the {@apilink BasicCrawler.run|`crawler.run()`} function.
@@ -661,6 +675,7 @@ export class BasicCrawler<
     private _experimentWarnings: Partial<Record<keyof CrawlerExperiments, boolean>> = {};
     private readonly crawlerId: string;
     private readonly hasExplicitId: boolean;
+    private readonly crawlerInstanceIndex: number;
     private readonly contextPipelineOptions: {
         contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;
         extendContext?: (context: Context) => Awaitable<ContextExtension>;
@@ -803,6 +818,7 @@ export class BasicCrawler<
             this.hasExplicitId = id !== undefined;
             // Store the user-provided ID, or generate a unique one for tracking purposes (not for state key)
             this.crawlerId = id ?? cryptoRandomObjectId();
+            this.crawlerInstanceIndex = BasicCrawler.instanceCount++;
 
             if (requestManager !== undefined) {
                 if (requestList !== undefined || requestQueue !== undefined) {
@@ -1284,19 +1300,21 @@ export class BasicCrawler<
             );
         }
 
-        const { purgeRequestQueue = true, ...addRequestsOptions } = options ?? {};
+        const { purgeRequestQueue, ...addRequestsOptions } = options ?? {};
 
         if (this.hasFinishedBefore) {
             // When executing the run method for the second time explicitly,
-            // we need to purge the default RQ to allow processing the same requests again - this is important so users can
+            // we need to purge the RQ to allow processing the same requests again — this is important so users can
             // pass in failed requests back to the `crawler.run()`, otherwise they would be considered as handled and
-            // ignored - as a failed requests is still handled.
-            const isDefaultQueue = this.requestQueue?.name === 'default';
-            if (isDefaultQueue && purgeRequestQueue && this.requestQueue) {
-                await this.requestQueue.drop();
-                this.requestQueue = await this._getRequestQueue();
-                this.requestManager = undefined;
-                await this.initializeRequestManager();
+            // ignored — as a failed request is still handled.
+            // By default (purgeRequestQueue unset or true), only the queue we created ourselves (ownedRequestQueue) is purged.
+            // When `purgeRequestQueue` is explicitly `true`, we also purge a user-supplied queue.
+            // When `purgeRequestQueue` is explicitly `false`, nothing is purged.
+            const shouldPurge = purgeRequestQueue !== false;
+            const queueToPurge = this.ownedRequestQueue ?? (purgeRequestQueue === true ? this.requestQueue : undefined);
+
+            if (queueToPurge && shouldPurge) {
+                await queueToPurge.purge();
                 this.handledRequestsCount = 0; // This would've been reset by this._init() further down below, but at that point `handledRequestsCount` could prevent `addRequests` from adding the initial requests
             }
 
@@ -1425,6 +1443,7 @@ export class BasicCrawler<
 
         if (!this.requestQueue) {
             this.requestQueue = await this._getRequestQueue();
+            this.ownedRequestQueue = this.requestQueue;
             this.requestManager = undefined;
         }
 
@@ -2262,6 +2281,11 @@ export class BasicCrawler<
     }
 
     private async _getRequestQueue() {
+        // The first crawler instance uses the default queue (null identifier);
+        // subsequent instances get their own queue via a unique alias so they don't collide.
+        const identifier =
+            this.crawlerInstanceIndex === 0 ? null : { alias: `__default_${this.crawlerInstanceIndex}__` };
+
         // Check if it's explicitly disabled
         // oxlint-disable-next-line typescript/no-deprecated -- still honored for opt-out until the flag is removed
         if (this.experiments.requestLocking === false) {
@@ -2272,10 +2296,10 @@ export class BasicCrawler<
                 this._experimentWarnings.requestLocking = true;
             }
 
-            return RequestQueueV1.open(null, { config: serviceLocator.getConfiguration() });
+            return RequestQueueV1.open(identifier, { config: serviceLocator.getConfiguration() });
         }
 
-        return RequestQueue.open(null, { config: serviceLocator.getConfiguration() });
+        return RequestQueue.open(identifier, { config: serviceLocator.getConfiguration() });
     }
 
     private requestMatchesEnqueueStrategy(request: Request) {
@@ -2349,9 +2373,14 @@ export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {}
 
 export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
     /**
-     * Whether to purge the RequestQueue before running the crawler again. Defaults to true, so it is possible to reprocess failed requests.
-     * When disabled, only new requests will be considered. Note that even a failed request is considered as handled.
-     * @default true
+     * Controls whether the request queue is purged between repeated `run()` calls on the same crawler instance.
+     * Purging clears all requests and resets internal counters, allowing the same URLs to be processed again.
+     *
+     * - **`undefined`** (default) — only the crawler's own (auto-created) queue is purged.
+     *   A user-supplied `requestQueue` is left untouched.
+     * - **`true`** — the queue is always purged, even if it was supplied by the user.
+     * - **`false`** — nothing is purged. Only genuinely new requests will be processed;
+     *   note that even a failed request is considered handled.
      */
     purgeRequestQueue?: boolean;
 }

packages/core/src/storages/request_provider.ts

@@ -99,6 +99,14 @@ export interface IRequestManager {
     addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;
 
     addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
+
+    /**
+     * Remove all requests from the queue but keep the queue itself, resetting it
+     * so it can be reused (e.g. across multiple `crawler.run()` calls).
+     *
+     * Implementations that do not support purging may leave this `undefined`.
+     */
+    purge?(): Promise<void>;
 }
 
 export abstract class RequestProvider implements IStorage, IRequestManager {
@@ -726,6 +734,28 @@ export abstract class RequestProvider implements IStorage, IRequestManager {
         serviceLocator.getStorageInstanceManager().removeFromCache(this);
     }
 
+    /**
+     * Remove all requests from the queue but keep the queue itself, resetting it
+     * so it can be reused (e.g. across multiple `crawler.run()` calls).
+     */
+    async purge(): Promise<void> {
+        checkStorageAccess();
+
+        await this.client.purge();
+
+        // Reset in-memory bookkeeping so the queue behaves as if freshly opened.
+        this.assumedTotalCount = 0;
+        this.assumedHandledCount = 0;
+        this.initialCount = 0;
+        this.initialHandledCount = 0;
+        this.queueHeadIds.clear();
+        this.requestCache.clear();
+        this.recentlyHandledRequestsCache.clear();
+        this.lastActivity = new Date();
+        this.isFinishedCalledWhileHeadWasNotEmpty = 0;
+        this.inProgressRequestBatchCount = 0;
+    }
+
     /**
      * @inheritdoc
      */

packages/core/src/storages/storage_instance_manager.ts

@@ -119,6 +119,33 @@ class StorageCache {
         }
     }
 
+    /**
+     * Ensure that the same string is not used as both a name and an alias for the same
+     * storage class + backend combination. Mirrors crawlee-python's `_check_name_alias_conflict`.
+     */
+    checkNameAliasConflict<T extends IStorage>(
+        cls: Constructor<T>,
+        { name, alias, clientCacheKey }: { name?: string; alias?: string; clientCacheKey: Hashable },
+    ): void {
+        if (alias) {
+            const existingByName = this.byName.get(cls)?.get(alias)?.get(clientCacheKey);
+            if (existingByName) {
+                throw new Error(
+                    `Cannot open storage with alias "${alias}" because a named storage with the same identifier already exists.`,
+                );
+            }
+        }
+        if (name) {
+            const existingByAlias = this.byAlias.get(cls)?.get(name)?.get(clientCacheKey);
+            if (existingByAlias) {
+                throw new Error(
+                    `Cannot open storage with name "${name}" because an alias storage with the same identifier already exists.` +
+                        ` If you meant to open the alias storage, use { alias: "${name}" } instead.`,
+                );
+            }
+        }
+    }
+
     /** Iterate all cached instances across all storage types. */
     *allValues(): IterableIterator<IStorage> {
         const seen = new Set<IStorage>();
@@ -222,6 +249,9 @@ export class StorageInstanceManager {
                 if (cached) return cached;
             }
 
+            // Prevent the same string from being used as both a name and an alias.
+            this.cache.checkNameAliasConflict(cls, { name, alias, clientCacheKey });
+
             // Cache miss — create the sub-client and storage instance.
             const subClient = await clientOpener();
             const storageInfo = await (
@@ -299,6 +329,7 @@ export interface DefaultStorageIdentifier {
  * - `string` → resolved via `storageExists` (ID-first, then name)
  * - `{ id }` → `{ id }`
  * - `{ name }` → `{ name }`
+ * - `{ alias }` → `{ alias }`
  */
 export async function resolveStorageIdentifier(
     identifier: string | StorageIdentifier | null | undefined,
@@ -324,6 +355,10 @@ export async function resolveStorageIdentifier(
         return { name: identifier.name };
     }
 
+    if ('alias' in identifier && identifier.alias) {
+        return { alias: identifier.alias };
+    }
+
     // Empty object — treated as default storage.
     return { alias: DEFAULT_STORAGE_ALIAS };
 }

packages/memory-storage/src/cache-helpers.ts

@@ -13,9 +13,12 @@ import { type MemoryStorage } from './memory-storage.js';
 const uuidRegex = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/i;
 
 export async function findOrCacheDatasetByPossibleId(client: MemoryStorage, entryNameOrId: string) {
-    // First check memory cache
+    // First check memory cache — match by id, name, or directoryName (which covers alias lookups)
     const found = client.datasetClientCache.find(
-        (store) => store.id === entryNameOrId || store.name?.toLowerCase() === entryNameOrId.toLowerCase(),
+        (store) =>
+            store.id === entryNameOrId ||
+            store.name?.toLowerCase() === entryNameOrId.toLowerCase() ||
+            store.directoryName.toLowerCase() === entryNameOrId.toLowerCase(),
     );
 
     if (found) {
@@ -116,9 +119,12 @@ export async function findOrCacheDatasetByPossibleId(client: MemoryStorage, entr
 }
 
 export async function findOrCacheKeyValueStoreByPossibleId(client: MemoryStorage, entryNameOrId: string) {
-    // First check memory cache
+    // First check memory cache — match by id, name, or directoryName (which covers alias lookups)
     const found = client.keyValueStoreCache.find(
-        (store) => store.id === entryNameOrId || store.name?.toLowerCase() === entryNameOrId.toLowerCase(),
+        (store) =>
+            store.id === entryNameOrId ||
+            store.name?.toLowerCase() === entryNameOrId.toLowerCase() ||
+            store.directoryName.toLowerCase() === entryNameOrId.toLowerCase(),
     );
 
     if (found) {
@@ -270,9 +276,12 @@ export async function findOrCacheKeyValueStoreByPossibleId(client: MemoryStorage
 }
 
 export async function findRequestQueueByPossibleId(client: MemoryStorage, entryNameOrId: string) {
-    // First check memory cache
+    // First check memory cache — match by id, name, or directoryName (which covers alias lookups)
     const found = client.requestQueueCache.find(
-        (store) => store.id === entryNameOrId || store.name?.toLowerCase() === entryNameOrId.toLowerCase(),
+        (store) =>
+            store.id === entryNameOrId ||
+            store.name?.toLowerCase() === entryNameOrId.toLowerCase() ||
+            store.directoryName.toLowerCase() === entryNameOrId.toLowerCase(),
     );
 
     if (found) {