fix: Rectify inconsistencies in the StorageClient interface family (#3800)

Author: janbuchar

docs/public-api/crawlee-core.api.md

@@ -363,6 +363,7 @@ export class Dataset<Data extends Dictionary = Dictionary> {
     reduce(iteratee: DatasetReducer<Data, Data>): Promise<Data | undefined>;
     reduce(iteratee: DatasetReducer<Data, Data>, memo: undefined, options: DatasetIteratorOptions): Promise<Data | undefined>;
     reduce<T>(iteratee: DatasetReducer<T, Data>, memo: T, options?: DatasetIteratorOptions): Promise<T>;
+    get stats(): DatasetStats;
     values(options?: DatasetIteratorOptions): AsyncIterable<Data> & Promise<Data[]>;
 }
 
@@ -447,6 +448,12 @@ export interface DatasetReducer<T, Data> {
     (memo: T, item: Data, index: number): Awaitable_2<T>;
 }
 
+// @public
+export interface DatasetStats {
+    readCount: number;
+    writeCount: number;
+}
+
 // @public
 export interface DefaultStorageIdentifier {
     // (undocumented)
@@ -740,7 +747,7 @@ export interface IRequestManager extends IRequestLoader {
     addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;
     purge?(): Promise<void>;
     reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
-    setExpectedRequestProcessingTimeSecs?(secs: number): void;
+    setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
 }
 
 // @internal (undocumented)
@@ -796,6 +803,7 @@ export class KeyValueStore {
     static recordExists(key: string): Promise<boolean>;
     setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
     static setValue<T>(key: string, value: T | null, options?: RecordOptions): Promise<void>;
+    get stats(): KeyValueStoreStats;
     values<T = unknown>(options?: KeyValueStoreIteratorOptions): AsyncIterable<T> & Promise<T[]>;
 }
 
@@ -822,6 +830,14 @@ export interface KeyValueStoreRawRecord {
     value: Buffer | ArrayBuffer;
 }
 
+// @public
+export interface KeyValueStoreStats {
+    deleteCount: number;
+    listCount: number;
+    readCount: number;
+    writeCount: number;
+}
+
 // @internal (undocumented)
 export type LoadedContext<Context extends RestrictedCrawlingContext> = IsAny<Context> extends true ? Context : {
     request: LoadedRequest<Context['request']>;
@@ -1213,7 +1229,7 @@ export class RequestManagerTandem implements IRequestManager {
     purge(): Promise<void>;
     // (undocumented)
     reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
-    setExpectedRequestProcessingTimeSecs(secs: number): void;
+    setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
 }
 
 // @public
@@ -1294,7 +1310,8 @@ export class RequestQueue implements IStorage, IRequestManager {
     reclaimRequest(request: Request_2, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo | null>;
     // (undocumented)
     protected requestCache: LruCache<RequestLruItem>;
-    setExpectedRequestProcessingTimeSecs(secs: number): void;
+    setExpectedRequestProcessingTimeSecs(secs: number): Promise<void>;
+    get stats(): RequestQueueStats;
     // (undocumented)
     timeoutSecs: number;
 }
@@ -1325,6 +1342,12 @@ export interface RequestQueueOptions {
     proxyConfiguration?: ProxyConfiguration;
 }
 
+// @public
+export interface RequestQueueStats {
+    headItemReadCount: number;
+    writeCount: number;
+}
+
 // @internal (undocumented)
 export const REQUESTS_PERSISTENCE_KEY = "REQUEST_LIST_REQUESTS";
 
@@ -1886,6 +1909,13 @@ export interface StorageOpenOptions {
     storageClient?: StorageClient;
 }
 
+// @public
+export class StorageStatsTracker<T extends Record<keyof T, number>> {
+    constructor(initial: T);
+    add(key: keyof T, by?: number): void;
+    get current(): T;
+}
+
 // @public
 export interface SystemInfo {
     // (undocumented)

docs/public-api/crawlee-types.api.md

@@ -113,10 +113,6 @@ export interface DatasetInfo {
     // (undocumented)
     accessedAt: Date;
     // (undocumented)
-    actId?: string;
-    // (undocumented)
-    actRunId?: string;
-    // (undocumented)
     createdAt: Date;
     // (undocumented)
     id: string;
@@ -128,18 +124,6 @@ export interface DatasetInfo {
     name?: string;
 }
 
-// @public (undocumented)
-export interface DatasetStats {
-    // (undocumented)
-    deleteCount?: number;
-    // (undocumented)
-    readCount?: number;
-    // (undocumented)
-    storageBytes?: number;
-    // (undocumented)
-    writeCount?: number;
-}
-
 // @public (undocumented)
 export type Dictionary<T = any> = Record<PropertyKey, T>;
 
@@ -230,7 +214,7 @@ export interface KeyValueStoreClient {
     getMetadata(): Promise<KeyValueStoreInfo>;
     getPublicUrl(key: string): Promise<string | undefined>;
     getValue(key: string): Promise<KeyValueStoreRecord | undefined>;
-    listKeys(options?: KeyValueStoreListKeysOptions): Promise<KeyValueStoreItemData[]>;
+    listKeys(options?: KeyValueStoreListKeysOptions): Promise<KeyValueStoreListKeysResult>;
     purge(): Promise<void>;
     recordExists(key: string): Promise<boolean>;
     setValue(record: KeyValueStoreInputRecord): Promise<void>;
@@ -241,21 +225,13 @@ export interface KeyValueStoreInfo {
     // (undocumented)
     accessedAt: Date;
     // (undocumented)
-    actId?: string;
-    // (undocumented)
-    actRunId?: string;
-    // (undocumented)
     createdAt: Date;
     // (undocumented)
     id: string;
     // (undocumented)
     modifiedAt: Date;
     // (undocumented)
     name?: string;
-    // (undocumented)
-    stats?: KeyValueStoreStats;
-    // (undocumented)
-    userId?: string;
 }
 
 // @public
@@ -270,6 +246,7 @@ export interface KeyValueStoreInputRecord {
 
 // @public (undocumented)
 export interface KeyValueStoreItemData {
+    contentType: string;
     // (undocumented)
     key: string;
     // (undocumented)
@@ -283,6 +260,16 @@ export interface KeyValueStoreListKeysOptions {
     prefix?: string;
 }
 
+// @public
+export interface KeyValueStoreListKeysResult {
+    count: number;
+    exclusiveStartKey?: string;
+    isTruncated: boolean;
+    items: KeyValueStoreItemData[];
+    limit: number;
+    nextExclusiveStartKey?: string;
+}
+
 // @public
 export interface KeyValueStoreRecord {
     // (undocumented)
@@ -296,20 +283,6 @@ export interface KeyValueStoreRecord {
 // @public
 export type KeyValueStoreRecordInputValue = Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;
 
-// @public (undocumented)
-export interface KeyValueStoreStats {
-    // (undocumented)
-    deleteCount?: number;
-    // (undocumented)
-    listCount?: number;
-    // (undocumented)
-    readCount?: number;
-    // (undocumented)
-    storageBytes?: number;
-    // (undocumented)
-    writeCount?: number;
-}
-
 // @public
 export interface NewPageOptions {
     id?: string;
@@ -321,7 +294,7 @@ export interface PageState {
     cookies: Cookie[];
 }
 
-// @public (undocumented)
+// @public
 export interface PaginatedList<Data> {
     count: number;
     desc?: boolean;
@@ -366,44 +339,28 @@ export type RedirectHandler = (redirectResponse: Response, updatedRequest: {
     headers: Headers;
 }) => void;
 
-// @public (undocumented)
-export interface RequestOptions {
-    // (undocumented)
-    [k: string]: unknown;
-    // (undocumented)
-    forefront?: boolean;
-}
-
 // @public
 export interface RequestQueueClient {
-    addBatchOfRequests(requests: RequestSchema[], options?: RequestOptions): Promise<BatchAddRequestsResult>;
+    addBatchOfRequests(requests: RequestSchema[], options?: RequestQueueOperationOptions): Promise<BatchAddRequestsResult>;
     drop(): Promise<void>;
-    fetchNextRequest(): Promise<RequestOptions | null>;
+    fetchNextRequest(): Promise<UpdateRequestSchema | undefined>;
     getMetadata(): Promise<RequestQueueInfo>;
-    getRequest(uniqueKey: string): Promise<RequestOptions | undefined>;
+    getRequest(uniqueKey: string): Promise<UpdateRequestSchema | undefined>;
     isEmpty(): Promise<boolean>;
     isFinished(): Promise<boolean>;
-    markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo | null>;
+    markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo | undefined>;
     purge(): Promise<void>;
-    reclaimRequest(request: UpdateRequestSchema, options?: RequestOptions): Promise<QueueOperationInfo | null>;
-    setExpectedRequestProcessingTimeSecs?(secs: number): void;
+    reclaimRequest(request: UpdateRequestSchema, options?: RequestQueueOperationOptions): Promise<QueueOperationInfo | undefined>;
+    setExpectedRequestProcessingTimeSecs?(secs: number): Promise<void>;
 }
 
 // @public (undocumented)
 export interface RequestQueueInfo {
     // (undocumented)
     accessedAt: Date;
     // (undocumented)
-    actId?: string;
-    // (undocumented)
-    actRunId?: string;
-    // (undocumented)
     createdAt: Date;
     // (undocumented)
-    expireAt?: string;
-    // (undocumented)
-    hadMultipleClients?: boolean;
-    // (undocumented)
     handledRequestCount: number;
     // (undocumented)
     id: string;
@@ -414,25 +371,12 @@ export interface RequestQueueInfo {
     // (undocumented)
     pendingRequestCount: number;
     // (undocumented)
-    stats?: RequestQueueStats;
-    // (undocumented)
     totalRequestCount: number;
-    // (undocumented)
-    userId?: string;
 }
 
-// @public (undocumented)
-export interface RequestQueueStats {
-    // (undocumented)
-    deleteCount?: number;
-    // (undocumented)
-    headItemReadCount?: number;
-    // (undocumented)
-    readCount?: number;
-    // (undocumented)
-    storageBytes?: number;
-    // (undocumented)
-    writeCount?: number;
+// @public
+export interface RequestQueueOperationOptions {
+    forefront?: boolean;
 }
 
 // @public (undocumented)

docs/upgrading/upgrading_v4.md

@@ -557,7 +557,7 @@ The sub-client interfaces (`DatasetClient`, `KeyValueStoreClient`, `RequestQueue
 | `setRecord(record, options?)` | `setValue(record)` |
 | `deleteRecord(key)` | `deleteValue(key)` |
 | `getRecordPublicUrl(key)` | `getPublicUrl(key)` |
-| `listKeys(options?)` → `KeyValueStoreClientListData` | `listKeys(options?)` → `KeyValueStoreItemData[]` |
+| `listKeys(options?)` → `KeyValueStoreClientListData` | `listKeys(options?)` → `KeyValueStoreListKeysResult` (a single self-describing page) |
 | `keys()`, `values()`, `entries()` | Removed (handled by `KeyValueStore` frontend) |
 
 **`RequestQueueClient`:**
@@ -580,9 +580,11 @@ The request queue client was reduced from 12 methods to 10. The distributed-lock
 
 The lifecycle is now: `fetchNextRequest()` hands out a pending request and marks it in progress; once processed, call `markRequestAsHandled(request)`; on failure call `reclaimRequest(request, { forefront? })` to return it to the queue.
 
+Methods that may have "nothing" to return now consistently resolve to `undefined` rather than `null`. `fetchNextRequest()` resolves to `undefined` when there is nothing to fetch, and `markRequestAsHandled()` / `reclaimRequest()` resolve to `undefined` when the request is not something the client is currently processing (a no-op, not an error). This matches the `undefined` already returned by `getRequest()`, `KeyValueStoreClient.getValue()`, and `getPublicUrl()`, so the whole client family uses a single "absent" sentinel. If you implemented a custom client that returned `null` from these methods, return `undefined` instead.
+
 `RequestQueueClient.isEmpty()` and `RequestQueueClient.isFinished()` answer two different questions:
 
-- `isEmpty()` is the weak check — `true` when the next `fetchNextRequest()` would return `null`, i.e. there is nothing left to fetch right now. Requests that are currently in progress (fetched but not yet handled or reclaimed) are **not** counted, because they are not fetchable. This is what drives the crawler's task scheduling.
+- `isEmpty()` is the weak check — `true` when the next `fetchNextRequest()` would return `undefined`, i.e. there is nothing left to fetch right now. Requests that are currently in progress (fetched but not yet handled or reclaimed) are **not** counted, because they are not fetchable. This is what drives the crawler's task scheduling.
 - `isFinished()` is the strong check — `true` only when there are no pending requests **and** no requests currently in progress (including those locked by other clients sharing the queue). This is what determines whether crawling is actually done. An in-progress request keeps the queue *empty but not finished*, which is what stops a crawler from shutting down while a request is still being processed.
 
 The separate `RequestQueueV1`/`RequestQueueV2` classes (and the `RequestProvider` base class) have been removed. They no longer differ in behavior — request coordination is now internal to the storage client — so they are merged into a single `RequestQueue` class. Replace any `RequestQueueV1`, `RequestQueueV2`, or `RequestProvider` imports with `RequestQueue`.
@@ -613,7 +615,9 @@ queue.setExpectedRequestProcessingTimeSecs(600);
 
 The `RequestQueue.internalTimeoutMillis` property and the associated "stuck queue" self-recovery have been removed. In v3 the `RequestQueue` frontend kept its own copy of the queue head and in-progress set, which could drift out of sync with the backing storage (an eventual-consistency hazard on the Apify platform); `isFinished()` watched for inactivity exceeding `internalTimeoutMillis` and reset that frontend state to recover. In v4 the frontend no longer holds any such bookkeeping — the storage client is the single source of truth — so there is nothing for a reset to fix, and stuck request locks now self-heal on expiry. Any consistency-recovery logic that is genuinely specific to the Apify platform's distributed storage belongs in the Apify SDK's client implementation instead, and is tracked in [apify/crawlee#3328](https://github.com/apify/crawlee/issues/3328).
 
-**Removed types** from `@crawlee/types`: `DatasetClientUpdateOptions`, `KeyValueStoreClientUpdateOptions`, `KeyValueStoreRecordOptions`, `KeyValueStoreClientListData`, `KeyValueStoreClientGetRecordOptions`, `QueueHead`, `RequestQueueHeadItem`, `ListOptions`, `ListAndLockOptions`, `ListAndLockHeadResult`, `ProlongRequestLockOptions`, `ProlongRequestLockResult`, `DeleteRequestLockOptions`. `KeyValueStoreClientListOptions` was renamed to `KeyValueStoreListKeysOptions`.
+**Apify-specific fields removed from storage metadata.** The metadata returned by `getMetadata()` (`DatasetInfo`, `KeyValueStoreInfo`, `RequestQueueInfo`) has been trimmed to what is meaningful for any storage backend. The following platform-specific fields were dropped: `actId`, `actRunId`, `userId`, and — on `RequestQueueInfo` — `expireAt` and `hadMultipleClients`. The per-storage `stats` field (and its `DatasetStats` / `KeyValueStoreStats` / `RequestQueueStats` types) was removed as well. If you consumed any of these, read them from the Apify API client directly; a custom `StorageClient` should simply stop returning them.
+
+**Removed types** from `@crawlee/types`: `DatasetClientUpdateOptions`, `KeyValueStoreClientUpdateOptions`, `KeyValueStoreRecordOptions`, `KeyValueStoreClientListData`, `KeyValueStoreClientGetRecordOptions`, `QueueHead`, `RequestQueueHeadItem`, `ListOptions`, `ListAndLockOptions`, `ListAndLockHeadResult`, `ProlongRequestLockOptions`, `ProlongRequestLockResult`, `DeleteRequestLockOptions`, `DatasetStats`, `KeyValueStoreStats`, `RequestQueueStats`. `KeyValueStoreClientListOptions` was renamed to `KeyValueStoreListKeysOptions`.
 
 The high-level storage classes (`Dataset`, `KeyValueStore`, `RequestQueue`) now receive their sub-client directly in the constructor options instead of receiving a `StorageClient` and calling its methods.
 

packages/basic-crawler/src/internals/basic-crawler.ts

@@ -582,6 +582,13 @@ export class BasicCrawler<
      */
     private ownedRequestManager?: IRequestManager;
 
+    /**
+     * Whether the request-processing-time hint has already been forwarded to the request manager. The hint
+     * derives only from `requestHandlerTimeoutMillis` (constant for the crawler's lifetime) and is raise-only,
+     * so it only needs to be applied once, at the first async access of the manager.
+     */
+    private requestManagerTimeoutsApplied = false;
+
     /**
      * A reference to the underlying {@apilink AutoscaledPool} class that manages the concurrency of the crawler.
      * > *NOTE:* This property is only initialized after calling the {@apilink BasicCrawler.run|`crawler.run()`} function.
@@ -858,11 +865,6 @@ export class BasicCrawler<
             this.internalTimeoutMillis =
                 tryEnv(process.env.CRAWLEE_INTERNAL_TIMEOUT) ?? Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
 
-            // override the default internal timeout of request queue to respect `requestHandlerTimeoutMillis`
-            if (this.requestManager !== undefined) {
-                this.applyRequestManagerTimeouts(this.requestManager);
-            }
-
             this.maxRequestRetries = maxRequestRetries;
             this.maxCrawlDepth = maxCrawlDepth;
             this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
@@ -1435,6 +1437,14 @@ export class BasicCrawler<
             this.requestManager = await this.openOwnedRequestQueue();
         }
 
+        // Apply the processing-time hint here (an async lifecycle point) rather than in the constructor,
+        // now that `setExpectedRequestProcessingTimeSecs` is async. The hint is raise-only and idempotent,
+        // but guard so we do not re-issue it on every call.
+        if (!this.requestManagerTimeoutsApplied) {
+            this.requestManagerTimeoutsApplied = true;
+            await this.applyRequestManagerTimeouts(this.requestManager);
+        }
+
         return this.requestManager;
     }
 
@@ -1458,7 +1468,6 @@ export class BasicCrawler<
             this.crawlerInstanceIndex === 0 ? null : { alias: `__default_${this.crawlerInstanceIndex}__` };
 
         const requestQueue = await RequestQueue.open(identifier, { config: serviceLocator.getConfiguration() });
-        this.applyRequestManagerTimeouts(requestQueue);
         this.ownedRequestManager = requestQueue;
         return requestQueue;
     }
@@ -1470,8 +1479,8 @@ export class BasicCrawler<
      * request from being handed out a second time while it is still being processed — and it works
      * regardless of whether the manager is a plain {@apilink RequestQueue} or a `RequestManagerTandem`.
      */
-    private applyRequestManagerTimeouts(requestManager: IRequestManager): void {
-        requestManager.setExpectedRequestProcessingTimeSecs?.(
+    private async applyRequestManagerTimeouts(requestManager: IRequestManager): Promise<void> {
+        await requestManager.setExpectedRequestProcessingTimeSecs?.(
             Math.max(this.requestHandlerTimeoutMillis / 1000 + 5, 60),
         );
     }