feat(utils): add sitemapFilter option to parseSitemap (#3557)

Author: sbruinsje

packages/utils/src/internals/sitemap.ts

@@ -194,6 +194,13 @@ export interface ParseSitemapOptions {
      * @default true
      */
     reportNetworkErrors?: boolean;
+    /**
+     * Optional filter for nested sitemap URLs discovered in sitemap index files.
+     * Called with the URL of each child sitemap before it is fetched.
+     * Return `true` to include the sitemap, `false` to skip it.
+     * If not provided, all nested sitemaps are followed.
+     */
+    nestedSitemapFilter?: (sitemapUrl: string) => boolean;
 }
 
 export async function* parseSitemap<T extends ParseSitemapOptions>(
@@ -209,6 +216,7 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
         sitemapRetries = 3,
         networkTimeouts,
         reportNetworkErrors = true,
+        nestedSitemapFilter,
     } = options ?? {};
 
     const sources = [...initialSources];
@@ -340,6 +348,11 @@ export async function* parseSitemap<T extends ParseSitemapOptions>(
 
         for await (const item of items) {
             if (item.type === 'sitemapUrl' && !visitedSitemapUrls.has(item.url)) {
+                if (nestedSitemapFilter && !nestedSitemapFilter(item.url)) {
+                    log.debug(`Skipping sitemap ${item.url} due to nestedSitemapFilter.`);
+                    continue;
+                }
+
                 sources.push({ type: 'url', url: item.url, depth: (source.depth ?? 0) + 1 });
                 if (emitNestedSitemaps) {
                     yield { loc: item.url, originSitemapUrl: null } as any;

packages/utils/test/sitemap.test.ts

@@ -294,6 +294,33 @@ describe('Sitemap', () => {
         );
     });
 
+    it('respects nestedSitemapFilter when following sitemap indexes', async () => {
+        const items: SitemapUrl[] = [];
+
+        for await (const item of parseSitemap(
+            [{ type: 'url', url: 'http://not-exists.com/sitemap_parent.xml' }],
+            undefined,
+            {
+                nestedSitemapFilter: (url) => !url.includes('sitemap_child_2'),
+            },
+        )) {
+            items.push(item);
+        }
+
+        expect(items).toHaveLength(5);
+        expect(items.every((item) => item.originSitemapUrl === 'http://not-exists.com/sitemap_child.xml')).toBe(true);
+    });
+
+    it('follows all nested sitemaps when nestedSitemapFilter is not provided', async () => {
+        const items: SitemapUrl[] = [];
+
+        for await (const item of parseSitemap([{ type: 'url', url: 'http://not-exists.com/sitemap_parent.xml' }])) {
+            items.push(item);
+        }
+
+        expect(items).toHaveLength(10);
+    });
+
     it('does not break on invalid xml', async () => {
         const sitemap = await Sitemap.load('http://not-exists.com/not_actual_xml.xml');
         expect(sitemap.urls).toEqual([]);