Skip to content

Commit e1d8c93

Browse files
authored
fix: Prevent browser-based crawling mode in AdaptivePlaywrightCrawler from swallowing logs (#3803)
closes #3447
1 parent 3b5fc41 commit e1d8c93

2 files changed

Lines changed: 56 additions & 1 deletion

File tree

packages/playwright-crawler/src/internals/adaptive-playwright-crawler.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -648,6 +648,7 @@ export class AdaptivePlaywrightCrawler<
648648
throw browserRun.error;
649649
}
650650

651+
browserRun.logs?.forEach(([log, method, ...args]) => log[method](...(args as [any, any])));
651652
await this.commitResult(crawlingContext, browserRun.result);
652653

653654
if (shouldDetectRenderingType) {

test/core/crawlers/adaptive_playwright_crawler.test.ts

Lines changed: 55 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,15 @@
11
import type { Server } from 'node:http';
22
import type { AddressInfo } from 'node:net';
33

4-
import { type Dictionary, EventType, KeyValueStore, serviceLocator } from '@crawlee/core';
4+
import {
5+
BaseCrawleeLogger,
6+
type CrawleeLogger,
7+
type CrawleeLoggerOptions,
8+
type Dictionary,
9+
EventType,
10+
KeyValueStore,
11+
serviceLocator,
12+
} from '@crawlee/core';
513
import type {
614
AdaptivePlaywrightCrawlerContext,
715
AdaptivePlaywrightCrawlerOptions,
@@ -14,6 +22,22 @@ import express from 'express';
1422
import { startExpressAppPromise } from '../../shared/_helper.js';
1523
import { MemoryStorageEmulator } from '../../shared/MemoryStorageEmulator.js';
1624

25+
// A minimal logger that records every message into a shared array. Child loggers share the same
26+
// array, so messages emitted by the crawler's prefixed child logger are captured as well.
27+
class RecordingLogger extends BaseCrawleeLogger {
28+
constructor(private readonly messages: string[]) {
29+
super();
30+
}
31+
32+
logWithLevel(_level: number, message: string): void {
33+
this.messages.push(message);
34+
}
35+
36+
protected createChild(_options: Partial<CrawleeLoggerOptions>): CrawleeLogger {
37+
return new RecordingLogger(this.messages);
38+
}
39+
}
40+
1741
describe('AdaptivePlaywrightCrawler', () => {
1842
// Set up an express server that will serve test pages
1943
const HOSTNAME = '127.0.0.1';
@@ -175,6 +199,36 @@ describe('AdaptivePlaywrightCrawler', () => {
175199
});
176200
});
177201

202+
test.each([['static'], ['clientOnly']] as const)(
203+
'should replay request handler logs (%s)',
204+
async (renderingType) => {
205+
const renderingTypePredictor = makeRiggedRenderingTypePredictor({
206+
detectionProbabilityRecommendation: 0,
207+
renderingType,
208+
});
209+
const url = new URL(`http://${HOSTNAME}:${port}/static`);
210+
211+
const messages: string[] = [];
212+
const requestHandler: AdaptivePlaywrightCrawlerOptions['requestHandler'] = vi.fn(async ({ log }) => {
213+
log.info('handler log message');
214+
});
215+
216+
const crawler = await makeOneshotCrawler(
217+
{
218+
requestHandler,
219+
renderingTypePredictor,
220+
logger: new RecordingLogger(messages),
221+
},
222+
[url.toString()],
223+
);
224+
225+
await crawler.run();
226+
227+
expect(requestHandler).toHaveBeenCalled();
228+
expect(messages).toContain('handler log message');
229+
},
230+
);
231+
178232
test('should not store detection results on non-detection runs', async () => {
179233
const renderingTypePredictor = makeRiggedRenderingTypePredictor({
180234
detectionProbabilityRecommendation: 0,

0 commit comments

Comments
 (0)