Skip to content

Commit 3b5fc41

Browse files
authored
fix: RequestManagerTandem returns correct total count (#3804)
Avoid double-counting requests in `RequestManagerTandem`. Closes #3787
1 parent 3365b2d commit 3b5fc41

2 files changed

Lines changed: 25 additions & 1 deletion

File tree

packages/core/src/storages/request_manager_tandem.ts

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,8 @@ export class RequestManagerTandem implements IRequestManager {
153153
const requestManager = await this.getRequestManager();
154154
const [managerTotal, loaderTotal] = await Promise.all([
155155
requestManager.getTotalCount(),
156-
this.requestLoader.getTotalCount(),
156+
// count only pending to avoid double counting, requests marked as "handled" have been moved to requestManager
157+
this.requestLoader.getPendingCount(),
157158
]);
158159
return managerTotal + loaderTotal;
159160
}

test/core/request_manager_tandem.test.ts

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -115,6 +115,29 @@ describe('RequestManagerTandem', () => {
115115
await expect(tandem.getHandledCount()).resolves.toBe(2);
116116
});
117117

118+
test('getTotalCount returns correct count', async () => {
119+
const requestList = await RequestList.open(null, [
120+
{ url: 'https://example.com/1' },
121+
{ url: 'https://example.com/2' },
122+
]);
123+
const requestQueue = await RequestQueue.open();
124+
const tandem = new RequestManagerTandem(requestList, requestQueue);
125+
126+
await expect(tandem.getTotalCount()).resolves.toBe(2);
127+
128+
const req = await tandem.fetchNextRequest();
129+
130+
await expect(tandem.getTotalCount()).resolves.toBe(2);
131+
132+
await tandem.reclaimRequest(req!);
133+
134+
await expect(tandem.getTotalCount()).resolves.toBe(2);
135+
136+
await tandem.addRequest({ url: 'https://example.com/3' });
137+
138+
await expect(tandem.getTotalCount()).resolves.toBe(3);
139+
});
140+
118141
test('isFinished returns true only when both list and queue are finished', async () => {
119142
const requestList = await RequestList.open(null, [{ url: 'https://example.com/1' }]);
120143
const requestQueue = await RequestQueue.open();

0 commit comments

Comments
 (0)