fix: filter sitemap-derived URLs by enqueue strategy (#3797)

`SitemapRequestList`, `Sitemap.load` / `parseSitemap`, and
`RobotsTxtFile.getSitemaps()` now apply an enqueue strategy to the URLs
they extract. They keep only entries that match the strategy (default
`same-hostname`) relative to their parent sitemap, and always drop
non-`http(s)` schemes. This brings sitemap loading in line with
`enqueueLinks`, which already scopes discovered links to the same
hostname by default.

The selected strategy is also stamped onto the emitted `Request`
objects, so it keeps being enforced after navigation (e.g. across
redirects).

This changes the default behavior: cross-host sitemap entries are no
longer enqueued unless you opt in with `enqueueStrategy: 'all'` (or
`'same-domain'` / `'same-origin'`).

Author: vdusek

packages/core/src/enqueue_links/enqueue_links.ts

@@ -1,5 +1,5 @@
 import type { BatchAddRequestsResult, Dictionary } from '@crawlee/types';
-import { type RobotsTxtFile } from '@crawlee/utils';
+import { EnqueueStrategy, type RobotsTxtFile } from '@crawlee/utils';
 import ow from 'ow';
 import { getDomain } from 'tldts';
 import type { SetRequired } from 'type-fest';
@@ -31,6 +31,8 @@ import {
     filterRequestsByPatterns,
 } from './shared';
 
+export { EnqueueStrategy };
+
 export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
     /** Limit the amount of actually enqueued URLs to this number. Useful for testing across the entire crawling scope. */
     limit?: number;
@@ -199,61 +201,6 @@ export interface EnqueueLinksOptions extends RequestQueueOperationOptions {
     onSkippedRequest?: SkippedRequestCallback;
 }
 
-/**
- * The different enqueueing strategies available.
- *
- * Depending on the strategy you select, we will only check certain parts of the URLs found. Here is a diagram of each URL part and their name:
- *
- * ```md
- * Protocol          Domain
- * ┌────┐          ┌─────────┐
- * https://example.crawlee.dev/...
- * │       └─────────────────┤
- * │             Hostname    │
- * │                         │
- * └─────────────────────────┘
- *          Origin
- *```
- *
- * - The `Protocol` is usually `http` or `https`
- * - The `Domain` represents the path without any possible subdomains to a website. For example, `crawlee.dev` is the domain of `https://example.crawlee.dev/`
- * - The `Hostname` is the full path to a website, including any subdomains. For example, `example.crawlee.dev` is the hostname of `https://example.crawlee.dev/`
- * - The `Origin` is the combination of the `Protocol` and `Hostname`. For example, `https://example.crawlee.dev` is the origin of `https://example.crawlee.dev/`
- */
-export enum EnqueueStrategy {
-    /**
-     * Matches any URLs found
-     */
-    All = 'all',
-
-    /**
-     * Matches any URLs that have the same hostname.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `https://example.com/hello` will not be matched.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    SameHostname = 'same-hostname',
-
-    /**
-     * Matches any URLs that have the same domain as the base URL.
-     * For example, `https://wow.an.example.com` and `https://example.com` will both be matched for a base url of
-     * `https://example.com`.
-     *
-     * > This strategy will match both `http` and `https` protocols regardless of the base URL protocol.
-     */
-    SameDomain = 'same-domain',
-
-    /**
-     * Matches any URLs that have the same hostname and protocol.
-     * For example, `https://wow.example.com/hello` will be matched for a base url of `https://wow.example.com/`, but
-     * `http://wow.example.com/hello` will not be matched.
-     *
-     * > This strategy will ensure the protocol of the base URL is the same as the protocol of the URL to be enqueued.
-     */
-    SameOrigin = 'same-origin',
-}
-
 /**
  * This function enqueues the urls provided to the {@apilink RequestQueue} provided. If you want to automatically find and enqueue links,
  * you should use the context-aware `enqueueLinks` function provided on the crawler contexts.

packages/core/src/storages/sitemap_request_list.ts

@@ -9,7 +9,7 @@ import defaultLog from '@apify/log';
 
 import { Configuration } from '../configuration';
 import type { GlobInput, RegExpInput, UrlPatternObject } from '../enqueue_links';
-import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps } from '../enqueue_links';
+import { constructGlobObjectsFromGlobs, constructRegExpObjectsFromRegExps, EnqueueStrategy } from '../enqueue_links';
 import { type EventManager, EventType } from '../events/event_manager';
 import { Request } from '../request';
 import { KeyValueStore } from './key_value_store';
@@ -95,6 +95,13 @@ export interface SitemapRequestListOptions extends UrlConstraints {
      * @default 200
      */
     maxBufferSize?: number;
+    /**
+     * Keep only sitemap-derived URLs matching this strategy relative to the parent sitemap URL; non-`http(s)`
+     * schemes are always dropped. The filtering stays enforced after navigation (e.g. across redirects).
+     * Pass `'all'` to disable host filtering.
+     * @default EnqueueStrategy.SameHostname
+     */
+    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
     /**
      * Advanced options for the underlying `parseSitemap` call.
      */
@@ -192,6 +199,11 @@ export class SitemapRequestList implements IRequestList {
      */
     private proxyUrl: string | undefined;
 
+    /**
+     * Enqueue strategy applied to sitemap-derived URLs and stamped onto the emitted `Request` objects.
+     */
+    private enqueueStrategy: EnqueueStrategy | `${EnqueueStrategy}`;
+
     /**
      * Logger instance.
      */
@@ -216,6 +228,7 @@ export class SitemapRequestList implements IRequestList {
                 signal: ow.optional.any(),
                 timeoutMillis: ow.optional.number,
                 maxBufferSize: ow.optional.number,
+                enqueueStrategy: ow.optional.string.oneOf(Object.values(EnqueueStrategy)),
                 parseSitemapOptions: ow.optional.object,
                 globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
                 exclude: ow.optional.array.ofType(
@@ -251,6 +264,7 @@ export class SitemapRequestList implements IRequestList {
         this.persistenceOptions = { enable: true, ...options.persistenceOptions };
 
         this.proxyUrl = options.proxyUrl;
+        this.enqueueStrategy = options.enqueueStrategy ?? EnqueueStrategy.SameHostname;
 
         this.urlQueueStream = this.createNewStream(options.maxBufferSize ?? 200);
 
@@ -382,6 +396,7 @@ export class SitemapRequestList implements IRequestList {
                     ...parseSitemapOptions,
                     maxDepth: 0,
                     emitNestedSitemaps: true,
+                    enqueueStrategy: this.enqueueStrategy,
                 })) {
                     if (!item.originSitemapUrl) {
                         // This is a nested sitemap
@@ -561,7 +576,7 @@ export class SitemapRequestList implements IRequestList {
             if (!nextUrl) {
                 return null;
             }
-            this.requestData.set(nextUrl, new Request({ url: nextUrl }));
+            this.requestData.set(nextUrl, new Request({ url: nextUrl, enqueueStrategy: this.enqueueStrategy }));
         }
 
         this.inProgress.add(nextUrl);

packages/utils/package.json

@@ -57,6 +57,7 @@
         "ow": "^0.28.1",
         "robots-parser": "^3.0.1",
         "sax": "^1.4.1",
+        "tldts": "^7.0.0",
         "tslib": "^2.4.0",
         "whatwg-mimetype": "^4.0.0"
     },

packages/utils/src/internals/robots.ts

@@ -3,11 +3,23 @@ import type { HTTPError as HTTPErrorClass } from 'got-scraping';
 import type { Robot } from 'robots-parser';
 import robotsParser from 'robots-parser';
 
+import log from '@apify/log';
+
 import { gotScraping } from './gotScraping';
 import { Sitemap } from './sitemap';
+import { type EnqueueStrategy, filterUrl } from './url';
 
 let HTTPError: typeof HTTPErrorClass;
 
+export interface RobotsTxtFileSitemapsOptions {
+    /**
+     * Keep only sitemap URLs matching this strategy relative to the robots.txt host; non-`http(s)` schemes
+     * are always dropped. Pass `'all'` to disable host filtering.
+     * @default 'same-hostname'
+     */
+    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
+}
+
 /**
  * Loads and queries information from a [robots.txt file](https://en.wikipedia.org/wiki/Robots.txt).
  *
@@ -28,6 +40,7 @@ let HTTPError: typeof HTTPErrorClass;
  */
 export class RobotsTxtFile {
     private constructor(
+        private url: string,
         private robots: Pick<Robot, 'isAllowed' | 'getSitemaps'>,
         private proxyUrl?: string,
     ) {}
@@ -59,7 +72,7 @@ export class RobotsTxtFile {
      * @param [proxyUrl] a proxy to be used for fetching the robots.txt file
      */
     static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile {
-        return new RobotsTxtFile(robotsParser(url, content), proxyUrl);
+        return new RobotsTxtFile(url, robotsParser(url, content), proxyUrl);
     }
 
     protected static async load(
@@ -81,10 +94,11 @@ export class RobotsTxtFile {
                 ...(options?.timeoutMillis ? { timeout: { request: options.timeoutMillis } } : {}),
             });
 
-            return new RobotsTxtFile(robotsParser(url.toString(), response.body), proxyUrl);
+            return new RobotsTxtFile(url, robotsParser(url.toString(), response.body), proxyUrl);
         } catch (e) {
             if (e instanceof HTTPError && e.response.statusCode === 404) {
                 return new RobotsTxtFile(
+                    url,
                     {
                         isAllowed() {
                             return true;
@@ -110,24 +124,41 @@ export class RobotsTxtFile {
     }
 
     /**
-     * Get URLs of sitemaps referenced in the robots file.
+     * Get URLs of sitemaps referenced in the robots file, filtered by `options.enqueueStrategy` relative to
+     * the robots.txt host (default `'same-hostname'`; pass `'all'` to disable). Non-`http(s)` schemes are
+     * always dropped.
      */
-    getSitemaps(): string[] {
-        return this.robots.getSitemaps();
+    getSitemaps(options: RobotsTxtFileSitemapsOptions = {}): string[] {
+        const { enqueueStrategy = 'same-hostname' } = options;
+        const sitemaps: string[] = [];
+
+        for (const sitemapUrl of this.robots.getSitemaps()) {
+            // `filterUrl` tolerates an unparseable origin (returns not-allowed) rather than throwing.
+            const { allowed, reason } = filterUrl(sitemapUrl, this.url, enqueueStrategy);
+            if (!allowed) {
+                log.warning(`Skipping sitemap ${sitemapUrl} listed in robots.txt at ${this.url}: ${reason}.`);
+                continue;
+            }
+            sitemaps.push(sitemapUrl);
+        }
+
+        return sitemaps;
     }
 
     /**
-     * Parse all the sitemaps referenced in the robots file.
+     * Parse all the sitemaps referenced in the robots file. `options` are forwarded to `getSitemaps`
+     * and the sitemap parser.
      */
-    async parseSitemaps(): Promise<Sitemap> {
-        return Sitemap.load(this.robots.getSitemaps(), this.proxyUrl);
+    async parseSitemaps(options: RobotsTxtFileSitemapsOptions = {}): Promise<Sitemap> {
+        return Sitemap.load(this.getSitemaps(options), this.proxyUrl, options);
     }
 
     /**
      * Get all URLs from all the sitemaps referenced in the robots file. A shorthand for `(await robots.parseSitemaps()).urls`.
+     * `options` are forwarded to `parseSitemaps`.
      */
-    async parseUrlsFromSitemaps(): Promise<string[]> {
-        return (await this.parseSitemaps()).urls;
+    async parseUrlsFromSitemaps(options: RobotsTxtFileSitemapsOptions = {}): Promise<string[]> {
+        return (await this.parseSitemaps(options)).urls;
     }
 }
 

packages/utils/src/internals/sitemap.ts

@@ -13,6 +13,7 @@ import log from '@apify/log';
 
 import { mergeAsyncIterables } from './iterables';
 import { RobotsFile } from './robots';
+import { type EnqueueStrategy, filterUrl } from './url';
 
 interface SitemapUrlData {
     loc: string;
@@ -206,6 +207,13 @@ export interface ParseSitemapOptions {
      * If not provided, all nested sitemaps are followed.
      */
     nestedSitemapFilter?: (sitemapUrl: string) => boolean;
+    /**
+     * Keep only sitemap-derived URLs (nested `<sitemap>` and `<url>` entries) matching this strategy
+     * relative to the parent sitemap URL; non-`http(s)` schemes are always dropped. Skipped for raw string
+     * sources (no parent URL). Pass `'all'` to disable host filtering.
+     * @default 'same-hostname'
+     */
+    enqueueStrategy?: EnqueueStrategy | `${EnqueueStrategy}`;
 }
 
 export async function* parseSitemap<T extends ParseSitemapOptions>(
@@ -222,6 +230,7 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
         networkTimeouts,
         reportNetworkErrors = true,
         nestedSitemapFilter,
+        enqueueStrategy = 'same-hostname',
     } = options ?? {};
 
     const sources = [...initialSources];
@@ -259,8 +268,11 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
 
         let items: AsyncIterable<SitemapItem> | null = null;
 
+        // Parent URL, parsed once and reused as the origin for the strategy checks below.
+        let sitemapUrl: URL | undefined;
+
         if (source.type === 'url') {
-            const sitemapUrl = new URL(source.url);
+            sitemapUrl = new URL(source.url);
             visitedSitemapUrls.add(sitemapUrl.toString());
             let retriesLeft = sitemapRetries + 1;
 
@@ -351,20 +363,44 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
             continue;
         }
 
+        // URL entries dropped by the enqueue strategy filter, reported in one warning per sitemap after
+        // the loop (per-entry warnings could flood the log; individual drops are logged at debug level).
+        let droppedUrlEntries = 0;
+
         for await (const item of items) {
             if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
                 if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
                     log.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
                     continue;
                 }
 
+                // Keep only nested sitemaps matching the strategy (and using http(s)) relative to the
+                // parent. Raw string sources have no parent URL, so the check is skipped.
+                if (source.type === 'url') {
+                    const { allowed, reason } = filterUrl(item.url, sitemapUrl!, enqueueStrategy);
+                    if (!allowed) {
+                        log.warning(`Skipping nested sitemap ${item.url} (parent ${source.url}): ${reason}.`);
+                        continue;
+                    }
+                }
+
                 sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
                 if (emitNestedSitemaps) {
                     yield { loc: item.url, originSitemapUrl: null } as any;
                 }
             }
 
             if (item.type === 'url') {
+                // Keep only URL entries that match the enqueue strategy relative to the parent (see above).
+                if (source.type === 'url') {
+                    const { allowed, reason } = filterUrl(item.loc, sitemapUrl!, enqueueStrategy);
+                    if (!allowed) {
+                        droppedUrlEntries++;
+                        log.debug(`Skipping sitemap URL ${item.loc} (parent ${source.url}): ${reason}.`);
+                        continue;
+                    }
+                }
+
                 yield {
                     ...item,
                     originSitemapUrl:
@@ -374,6 +410,12 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
                 };
             }
         }
+
+        if (droppedUrlEntries > 0 && source.type === 'url') {
+            log.warning(
+                `Skipped ${droppedUrlEntries} URL(s) from sitemap ${source.url} not matching enqueue strategy '${enqueueStrategy}' (or using a non-http(s) scheme). Enable debug logs to see each skipped URL.`,
+            );
+        }
     }
 }
 
@@ -546,7 +588,8 @@ export async function* discoverValidSitemaps(
                 timeoutMillis: requestTimeoutMillis,
                 signal,
             });
-            for (const sitemapUrl of robotsFile.getSitemaps()) {
+            // Surface all referenced sitemaps, including cross-host; scoping happens at load time.
+            for (const sitemapUrl of robotsFile.getSitemaps({ enqueueStrategy: 'all' })) {
                 if (addSitemapUrl(sitemapUrl)) {
                     yield sitemapUrl;
                 }