refactor!: Rename StorageManager to StorageInstanceManager and use it to centralize storage instance caching

packages/core/src/crawlers/crawler_commons.ts

@@ -9,7 +9,7 @@ import type { Session } from '../session_pool/session.js';
 import type { Dataset } from '../storages/dataset.js';
 import { KeyValueStore, type RecordOptions } from '../storages/key_value_store.js';
 import type { RequestQueueOperationOptions } from '../storages/request_provider.js';
-import type { StorageIdentifier } from '../storages/storage_manager.js';
+import type { StorageIdentifier } from '../storages/storage_instance_manager.js';
 
 /** @internal */
 export type IsAny<T> = 0 extends 1 & T ? true : false;

packages/core/src/service_locator.ts

@@ -11,8 +11,7 @@ import type { EventManager } from './events/event_manager.js';
 import { LocalEventManager } from './events/local_event_manager.js';
 import type { CrawleeLogger } from './log.js';
 import { ApifyLogAdapter } from './log.js';
-import type { IStorage, StorageManager } from './storages/storage_manager.js';
-import type { Constructor } from './typedefs.js';
+import { StorageInstanceManager } from './storages/storage_instance_manager.js';
 
 interface ServiceLocatorInterface {
     /**
@@ -77,12 +76,13 @@ interface ServiceLocatorInterface {
      */
     getChildLog(prefix: string): CrawleeLogger;
 
-    getStorageManager(constructor: Constructor<IStorage>): StorageManager | undefined;
-
-    setStorageManager(constructor: Constructor<IStorage>, storageManager: StorageManager): void;
+    /**
+     * Get the storage instance manager (shared across all storage types).
+     */
+    getStorageInstanceManager(): StorageInstanceManager;
 
     /**
-     * Clears all storage manager caches.
+     * Clears all storage instance manager caches.
      * @internal
      */
     clearStorageManagerCache(): void;
@@ -134,10 +134,11 @@ export class ServiceLocator implements ServiceLocatorInterface {
     private logger?: CrawleeLogger;
 
     /**
-     * Storage managers for Dataset, KeyValueStore, and RequestQueue.
-     * Manages caching and lifecycle of storage instances.
+     * Unified storage instance manager for Dataset, KeyValueStore, and RequestQueue.
+     * Shared across all ServiceLocator instances (global singleton), matching crawlee-python.
+     * Per-crawler isolation is achieved via `clientCacheKey`, not separate manager instances.
      */
-    private storageManagers = new Map<Constructor<IStorage>, StorageManager>();
+    private static storageInstanceManager?: StorageInstanceManager;
 
     /**
      * Creates a new ServiceLocator instance.
@@ -264,34 +265,25 @@ export class ServiceLocator implements ServiceLocatorInterface {
         return this.getLogger().child({ prefix });
     }
 
-    getStorageManager(constructor: Constructor<IStorage>): StorageManager | undefined {
-        return this.storageManagers.get(constructor);
-    }
-
-    setStorageManager(constructor: Constructor<IStorage>, storageManager: StorageManager): void {
-        if (this.storageManagers.has(constructor)) {
-            throw new ServiceConflictError(
-                `StorageManager(${constructor.name})`,
-                storageManager,
-                this.storageManagers.get(constructor),
-            );
+    getStorageInstanceManager(): StorageInstanceManager {
+        if (!ServiceLocator.storageInstanceManager) {
+            ServiceLocator.storageInstanceManager = new StorageInstanceManager();
         }
-
-        this.storageManagers.set(constructor, storageManager);
+        return ServiceLocator.storageInstanceManager;
     }
 
     clearStorageManagerCache(): void {
-        this.storageManagers.forEach((manager) => {
-            // KeyValueStore has a clearCache method on its instances
-            // TODO this uses fragile string matching and `any` casts into private fields - remove as part of
-            //  https://github.com/apify/crawlee/issues/3075 (Storage instance management will be reworked significantly)
-            if ((manager as any).name === 'KeyValueStore') {
-                (manager as any).cache?.forEach((item: any) => {
-                    item.clearCache?.();
-                });
+        if (ServiceLocator.storageInstanceManager) {
+            // KeyValueStore instances have a clearCache method — call it on any cached
+            // instance that exposes it, without importing KeyValueStore (avoids circular deps).
+            for (const instance of ServiceLocator.storageInstanceManager.getAllInstances()) {
+                if ('clearCache' in instance && typeof (instance as any).clearCache === 'function') {
+                    (instance as any).clearCache();
+                }
             }
-        });
-        this.storageManagers.clear();
+
+            ServiceLocator.storageInstanceManager.clearCache();
+        }
     }
 
     reset(): void {
@@ -300,6 +292,7 @@ export class ServiceLocator implements ServiceLocatorInterface {
         this.storageClient = undefined;
         this.logger = undefined;
         this.clearStorageManagerCache();
+        ServiceLocator.storageInstanceManager = undefined;
     }
 }
 

packages/core/src/storages/dataset.ts

@@ -10,8 +10,7 @@ import { serviceLocator } from '../service_locator.js';
 import type { Awaitable } from '../typedefs.js';
 import { checkStorageAccess } from './access_checking.js';
 import { KeyValueStore } from './key_value_store.js';
-import type { StorageIdentifier, StorageManagerOptions } from './storage_manager.js';
-import { StorageManager } from './storage_manager.js';
+import type { StorageIdentifier, StorageManagerOptions } from './storage_instance_manager.js';
 import { purgeDefaultStorages } from './utils.js';
 
 /** @internal */
@@ -681,8 +680,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {
         checkStorageAccess();
 
         await this.client.delete();
-        const manager = StorageManager.getManager(Dataset);
-        manager.closeStorage(this);
+        serviceLocator.getStorageInstanceManager().removeFromCache(this);
     }
 
     /**
@@ -715,13 +713,19 @@ export class Dataset<Data extends Dictionary = Dictionary> {
         );
 
         options.config ??= Configuration.getGlobalConfig();
-        options.storageClient ??= serviceLocator.getStorageClient();
 
-        await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });
+        const client = options.storageClient ?? serviceLocator.getStorageClient();
 
-        const manager = StorageManager.getManager<Dataset<Data>>(this);
+        await purgeDefaultStorages({ onlyPurgeOnce: true, client, config: options.config });
 
-        return manager.openStorage(identifier, options.storageClient);
+        return serviceLocator.getStorageInstanceManager().openStorage<Dataset<Data>>(this, {
+            storageTypeName: 'Dataset',
+            identifier: identifier ?? null,
+            client,
+            config: options.config,
+            clientOpener: (resolved) => client.createDatasetClient(resolved),
+            clientCacheKey: client.getStorageClientCacheKey?.() ?? client.constructor.name,
+        });
     }
 
     /**

packages/core/src/storages/index.ts

@@ -6,7 +6,7 @@ export * from './request_provider.js';
 export { RequestQueueV1 } from './request_queue.js';
 export { RequestQueue } from './request_queue_v2.js';
 export { RequestQueue as RequestQueueV2 } from './request_queue_v2.js';
-export * from './storage_manager.js';
+export * from './storage_instance_manager.js';
 export * from './utils.js';
 export * from './access_checking.js';
 export * from './sitemap_request_list.js';

packages/core/src/storages/key_value_store.ts

@@ -12,8 +12,7 @@ import { Configuration } from '../configuration.js';
 import { serviceLocator } from '../service_locator.js';
 import type { Awaitable } from '../typedefs.js';
 import { checkStorageAccess } from './access_checking.js';
-import type { StorageIdentifier, StorageManagerOptions } from './storage_manager.js';
-import { StorageManager } from './storage_manager.js';
+import type { StorageIdentifier, StorageManagerOptions } from './storage_instance_manager.js';
 import { purgeDefaultStorages } from './utils.js';
 
 /**
@@ -417,8 +416,7 @@ export class KeyValueStore {
         checkStorageAccess();
 
         await this.client.delete();
-        const manager = StorageManager.getManager(KeyValueStore);
-        manager.closeStorage(this);
+        serviceLocator.getStorageInstanceManager().removeFromCache(this);
     }
 
     /** @internal */
@@ -612,13 +610,18 @@ export class KeyValueStore {
         );
 
         options.config ??= Configuration.getGlobalConfig();
-        options.storageClient ??= serviceLocator.getStorageClient();
+        const client = options.storageClient ?? serviceLocator.getStorageClient();
 
-        await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });
+        await purgeDefaultStorages({ onlyPurgeOnce: true, client, config: options.config });
 
-        const manager = StorageManager.getManager(this);
-
-        return manager.openStorage(identifier, options.storageClient);
+        return serviceLocator.getStorageInstanceManager().openStorage<KeyValueStore>(this, {
+            storageTypeName: 'KeyValueStore',
+            identifier: identifier ?? null,
+            client,
+            config: options.config,
+            clientOpener: (resolved) => client.createKeyValueStoreClient(resolved),
+            clientCacheKey: client.getStorageClientCacheKey?.() ?? client.constructor.name,
+        });
     }
 
     /**

packages/core/src/storages/request_provider.ts

@@ -32,10 +32,8 @@ import type { ProxyConfiguration } from '../proxy_configuration.js';
 import type { InternalSource, RequestOptions, Source } from '../request.js';
 import { Request } from '../request.js';
 import { serviceLocator } from '../service_locator.js';
-import type { Constructor } from '../typedefs.js';
 import { checkStorageAccess } from './access_checking.js';
-import type { IStorage, StorageIdentifier, StorageManagerOptions } from './storage_manager.js';
-import { StorageManager } from './storage_manager.js';
+import type { IStorage, StorageIdentifier, StorageManagerOptions } from './storage_instance_manager.js';
 import { getRequestId, purgeDefaultStorages, QUERY_HEAD_MIN_LENGTH } from './utils.js';
 
 export type RequestsLike = AsyncIterable<Source | string> | Iterable<Source | string> | (Source | string)[];
@@ -120,6 +118,7 @@ export abstract class RequestProvider implements IStorage, IRequestManager {
 
     private initialCount = 0;
     private initialHandledCount = 0; // We track this separately from `assumedHandledCount` which is used non-trivially by RequestQueueV1
+    private isInitialized = false;
 
     protected queueHeadIds = new ListDictionary<string>();
     protected requestCache: LruCache<RequestLruItem>;
@@ -722,8 +721,7 @@ export abstract class RequestProvider implements IStorage, IRequestManager {
         checkStorageAccess();
 
         await this.client.delete();
-        const manager = StorageManager.getManager(this.constructor as Constructor<IStorage>);
-        manager.closeStorage(this);
+        serviceLocator.getStorageInstanceManager().removeFromCache(this);
     }
 
     /**
@@ -875,30 +873,42 @@ export abstract class RequestProvider implements IStorage, IRequestManager {
             }),
         );
 
-        options.storageClient ??= serviceLocator.getStorageClient();
+        const client = options.storageClient ?? serviceLocator.getStorageClient();
+        const config = options.config ?? serviceLocator.getConfiguration();
 
-        await purgeDefaultStorages({ onlyPurgeOnce: true, client: options.storageClient, config: options.config });
+        await purgeDefaultStorages({ onlyPurgeOnce: true, client, config });
 
-        const manager = StorageManager.getManager(this as typeof BuiltRequestProvider);
-        const queue = await manager.openStorage(identifier, options.storageClient);
+        const queue = await serviceLocator
+            .getStorageInstanceManager()
+            .openStorage<RequestProvider>(this as typeof BuiltRequestProvider, {
+                storageTypeName: 'RequestQueue',
+                identifier: identifier ?? null,
+                client,
+                config,
+                clientOpener: (resolved) => client.createRequestQueueClient(resolved),
+                clientCacheKey: client.getStorageClientCacheKey?.() ?? client.constructor.name,
+            });
         queue.proxyConfiguration = options.proxyConfiguration;
+        queue.httpClient = options.httpClient;
 
-        // Re-create the request queue client with clientKey and timeoutSecs so that
-        // request locking works correctly for API-backed implementations.
-        // TODO: clientKey/timeoutSecs are Apify-platform concerns and should eventually be pushed
-        // down into the Apify SDK's client implementation, aligning with crawlee-python's approach
-        // where locking is handled internally by the client (see crawlee-python PR #1194).
-        queue.client = await options.storageClient.createRequestQueueClient({
-            id: queue.id,
-            clientKey: queue.clientKey,
-            timeoutSecs: queue.timeoutSecs,
-        });
+        if (!queue.isInitialized) {
+            // Re-create the request queue client with clientKey and timeoutSecs so that
+            // request locking works correctly for API-backed implementations.
+            // TODO: clientKey/timeoutSecs are Apify-platform concerns and should eventually be pushed
+            // down into the Apify SDK's client implementation, aligning with crawlee-python's approach
+            // where locking is handled internally by the client (see crawlee-python PR #1194).
+            queue.client = await client.createRequestQueueClient({
+                id: queue.id,
+                clientKey: queue.clientKey,
+                timeoutSecs: queue.timeoutSecs,
+            });
 
-        const queueInfo = await queue.client.getMetadata();
+            const queueInfo = await queue.client.getMetadata();
 
-        queue.initialCount = queueInfo.totalRequestCount;
-        queue.initialHandledCount = queueInfo.handledRequestCount;
-        queue.httpClient = options.httpClient;
+            queue.initialCount = queueInfo.totalRequestCount;
+            queue.initialHandledCount = queueInfo.handledRequestCount;
+            queue.isInitialized = true;
+        }
 
         return queue;
     }