fix(core): ensure maxCrawlDepth warning is logged only once (#3337)

B4nan · claude · web-flow · commit 9d013342c539 · 2026-01-14T15:07:43.000+01:00
## Summary - Refactored enqueueLinks limit logging to distinguish between two scenarios: - **`enqueueLinks({ limit: X })`**: Now logs once per request handler call (not once per run), providing feedback each time the limit is hit in different handlers - **`maxCrawlDepth`**: Logs once per crawler run, avoiding spam when multiple handlers trigger the depth limit - Replaced multiple boolean flags with a cleaner `Set<string>` pattern for "log once per run" messages - Added comprehensive tests for both logging behaviors ## Test plan - [x] Existing tests pass - [x] New test: `enqueueLinks limit log message should be logged once per request handler, not once per run` - [x] New test: `maxCrawlDepth limit log message should only be logged once per run` 🤖 Generated with [Claude Code](https://claude.com/claude-code) Closes #3336 --------- Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -563,8 +563,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
     protected respectRobotsTxtFile: boolean | { userAgent?: string };
     protected onSkippedRequest?: SkippedRequestCallback;
     private _closeEvents?: boolean;
-    private shouldLogMaxEnqueuedRequestsExceeded = true;
-    private shouldLogShuttingDown = true;
+    private loggedPerRun = new Set<string>();
     private experiments: CrawlerExperiments;
     private readonly robotsTxtFileCache: LruCache<RobotsTxtFile>;
     private _experimentWarnings: Partial<Record<keyof CrawlerExperiments, boolean>> = {};
@@ -811,24 +810,20 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
             runTaskFunction: this._runTaskFunction.bind(this),
             isTaskReadyFunction: async () => {
                 if (isMaxPagesExceeded()) {
-                    if (this.shouldLogShuttingDown) {
-                        log.info(
-                            'Crawler reached the maxRequestsPerCrawl limit of ' +
-                                `${this.maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`,
-                        );
-                        this.shouldLogShuttingDown = false;
-                    }
+                    this.logOncePerRun(
+                        'shuttingDown',
+                        'Crawler reached the maxRequestsPerCrawl limit of ' +
+                            `${this.maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`,
+                    );
                     return false;
                 }
 
                 if (this.unexpectedStop) {
-                    if (this.shouldLogShuttingDown) {
-                        this.log.info(
-                            'No new requests are allowed because the `stop()` method has been called. ' +
-                                'Ongoing requests will be allowed to complete.',
-                        );
-                        this.shouldLogShuttingDown = false;
-                    }
+                    this.logOncePerRun(
+                        'shuttingDown',
+                        'No new requests are allowed because the `stop()` method has been called. ' +
+                            'Ongoing requests will be allowed to complete.',
+                    );
                     return false;
                 }
 
@@ -841,15 +836,13 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
                             'and all requests that were in progress at that time have now finished. ' +
                             `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`,
                     );
-                    this.shouldLogShuttingDown = false;
                     return true;
                 }
 
                 if (this.unexpectedStop) {
                     this.log.info(
                         'The crawler has finished all the remaining ongoing requests and will shut down now.',
                     );
-                    this.shouldLogShuttingDown = false;
                     return true;
                 }
 
@@ -1000,8 +993,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
 
         this.unexpectedStop = false;
         this.running = true;
-        this.shouldLogMaxEnqueuedRequestsExceeded = true;
-        this.shouldLogShuttingDown = true;
+        this.loggedPerRun.clear();
 
         await purgeDefaultStorages({
             onlyPurgeOnce: true,
@@ -1154,24 +1146,31 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
     }
 
     protected async handleSkippedRequest(options: Parameters<SkippedRequestCallback>[0]): Promise<void> {
-        if (options.reason === 'limit' && this.shouldLogMaxEnqueuedRequestsExceeded) {
-            this.log.info(
+        if (options.reason === 'limit') {
+            this.logOncePerRun(
+                'maxRequestsPerCrawl',
                 'The number of requests enqueued by the crawler reached the maxRequestsPerCrawl limit of ' +
                     `${this.maxRequestsPerCrawl} requests and no further requests will be added.`,
             );
-            this.shouldLogMaxEnqueuedRequestsExceeded = false;
         }
 
-        if (options.reason === 'enqueueLimit') {
-            const enqueuedRequestLimit = this.calculateEnqueuedRequestLimit();
-            if (enqueuedRequestLimit === undefined || enqueuedRequestLimit !== 0) {
-                this.log.info('The number of requests enqueued by the crawler reached the enqueueLinks limit.');
-            }
+        if (options.reason === 'depth') {
+            this.logOncePerRun(
+                'maxCrawlDepth',
+                `The crawler reached the maxCrawlDepth limit of ${this.maxCrawlDepth} and no further requests will be enqueued.`,
+            );
         }
 
         await this.onSkippedRequest?.(options);
     }
 
+    private logOncePerRun(key: string, message: string): void {
+        if (!this.loggedPerRun.has(key)) {
+            this.log.info(message);
+            this.loggedPerRun.add(key);
+        }
+    }
+
     /**
      * Adds requests to the queue in batches. By default, it will resolve after the initial batch is added, and continue
      * adding the rest in background. You can configure the batch size via `batchSize` option and the sleep time in between
@@ -1712,10 +1711,26 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
             return options.transformRequestFunction?.(newRequest) ?? newRequest;
         };
 
+        // Create a request-scoped callback that logs enqueueLimit once per request handler call
+        // Only log if an explicit limit was passed to enqueueLinks (not the internal maxRequestsPerCrawl-derived limit)
+        let loggedEnqueueLimitForThisRequest = false;
+        const onSkippedRequest: SkippedRequestCallback = async (skippedOptions) => {
+            if (skippedOptions.reason === 'enqueueLimit') {
+                if (!loggedEnqueueLimitForThisRequest && options.limit !== undefined) {
+                    this.log.info(
+                        `Skipping URLs in the handler for ${request.url} due to the enqueueLinks limit of ${options.limit}.`,
+                    );
+                    loggedEnqueueLimitForThisRequest = true;
+                }
+            }
+
+            await this.handleSkippedRequest(skippedOptions);
+        };
+
         return enqueueLinks({
             requestQueue,
             robotsTxtFile: await this.getRobotsTxtFileForUrl(request!.url),
-            onSkippedRequest: this.handleSkippedRequest,
+            onSkippedRequest,
             limit: this.calculateEnqueuedRequestLimit(options.limit),
 
             // Allow user options to override defaults set above ⤴
diff --git a/test/core/crawlers/basic_crawler.test.ts b/test/core/crawlers/basic_crawler.test.ts
@@ -1809,6 +1809,131 @@ describe('BasicCrawler', () => {
             // Should only have added the first 2 requests (since 2 were already processed and 1 is in progress, limit allows 2 more)
             expect(addRequestsBatchedSpy).toHaveBeenCalledTimes(2);
         });
+
+        test('enqueueLinks limit log message should only be logged once', async () => {
+            const requestQueue = await RequestQueue.open();
+
+            // Will try to add 10 requests with a limit of 2
+            const requestsToAdd = Array.from({ length: 10 }, (_, i) => `http://example.com/${i}`);
+
+            const crawler = new BasicCrawler({
+                requestQueue,
+                requestHandler: async (context) => {
+                    if (context.request.label) {
+                        return;
+                    }
+
+                    await context.enqueueLinks({ urls: requestsToAdd, limit: 2, label: 'child' });
+                },
+            });
+
+            const infoSpy = vitest.spyOn(crawler.log, 'info');
+
+            await crawler.run(['http://example.com']);
+
+            // The enqueueLinks limit message should only appear once, not 8 times (for each skipped request)
+            const enqueueLimitMessages = infoSpy.mock.calls.filter(
+                (call) => typeof call[0] === 'string' && call[0].includes('enqueueLinks limit'),
+            );
+            expect(enqueueLimitMessages).toHaveLength(1);
+        });
+
+        test('enqueueLinks limit log message should be logged again on subsequent runs', async () => {
+            const requestQueue = await RequestQueue.open();
+
+            const requestsToAdd = Array.from({ length: 5 }, (_, i) => `http://example.com/${i}`);
+
+            const crawler = new BasicCrawler({
+                requestQueue,
+                requestHandler: async (context) => {
+                    if (context.request.label) {
+                        return;
+                    }
+
+                    await context.enqueueLinks({ urls: requestsToAdd, limit: 1, label: 'child' });
+                },
+            });
+
+            const infoSpy = vitest.spyOn(crawler.log, 'info');
+
+            // First run
+            await crawler.run(['http://example.com/first']);
+
+            // Second run with a new start URL
+            await crawler.run(['http://example.com/second']);
+
+            // The enqueueLinks limit message should appear twice (once per run)
+            const enqueueLimitMessages = infoSpy.mock.calls.filter(
+                (call) => typeof call[0] === 'string' && call[0].includes('enqueueLinks limit'),
+            );
+            expect(enqueueLimitMessages).toHaveLength(2);
+        });
+
+        test('enqueueLinks limit log message should be logged once per request handler, not once per run', async () => {
+            const requestQueue = await RequestQueue.open();
+
+            // Each handler will try to add 5 URLs with a limit of 1
+            const requestsToAdd = Array.from({ length: 5 }, (_, i) => `http://example.com/child${i}`);
+
+            const crawler = new BasicCrawler({
+                requestQueue,
+                requestHandler: async (context) => {
+                    if (context.request.label === 'child') {
+                        return;
+                    }
+
+                    await context.enqueueLinks({ urls: requestsToAdd, limit: 1, label: 'child' });
+                },
+            });
+
+            const infoSpy = vitest.spyOn(crawler.log, 'info');
+
+            // Single run with two initial requests - both will trigger the limit
+            await crawler.run(['http://example.com/first', 'http://example.com/second']);
+
+            // The enqueueLinks limit message should appear twice (once per request handler that triggered the limit)
+            const enqueueLimitMessages = infoSpy.mock.calls.filter(
+                (call) => typeof call[0] === 'string' && call[0].includes('enqueueLinks limit'),
+            );
+            expect(enqueueLimitMessages).toHaveLength(2);
+        });
+
+        test('maxCrawlDepth limit log message should only be logged once per run', async () => {
+            const requestQueue = await RequestQueue.open();
+
+            // Each handler will try to add URLs that exceed maxCrawlDepth
+            const requestsToAdd = Array.from({ length: 5 }, (_, i) => `http://example.com/child${i}`);
+
+            const crawler = new BasicCrawler({
+                requestQueue,
+                maxCrawlDepth: 1, // Only allow depth 0 (initial) and depth 1 (first level children)
+                requestHandler: async (context) => {
+                    // Stop processing at depth 2 to avoid infinite loops
+                    if (context.request.crawlDepth >= 2) {
+                        return;
+                    }
+
+                    // This will add requests at depth+1, so initial requests add at depth 1 (allowed)
+                    // and depth 1 requests add at depth 2 (blocked by maxCrawlDepth)
+                    await context.enqueueLinks({
+                        urls: requestsToAdd,
+                        label: `depth-${context.request.crawlDepth + 1}`,
+                    });
+                },
+            });
+
+            const infoSpy = vitest.spyOn(crawler.log, 'info');
+
+            // Run with two initial requests
+            // Each will enqueue children at depth 1, then those children will try to enqueue at depth 2 (blocked)
+            await crawler.run(['http://example.com/first', 'http://example.com/second']);
+
+            // The maxCrawlDepth limit message should only appear once per run, even though multiple requests triggered it
+            const maxCrawlDepthMessages = infoSpy.mock.calls.filter(
+                (call) => typeof call[0] === 'string' && call[0].includes('maxCrawlDepth'),
+            );
+            expect(maxCrawlDepthMessages).toHaveLength(1);
+        });
     });
 
     describe('addRequests input validation', () => {
