fix(utils): allow apostrophes and asterisks in extracted URLs (#3790)

Closes #2755

`RequestList`/`requestsFromUrl` extracts URLs from the fetched body with
`extractUrls`, which uses `URL_NO_COMMAS_REGEX` /
`URL_WITH_COMMAS_REGEX`. The path/query character class of those regexes
was missing `'` and `*`, both valid RFC 3986 path sub-delims, so a URL
was truncated at the first apostrophe or asterisk. The Zillow example
from the issue, `.../141-O'Canoe-Pl-Hertford-NC-27944/...`, came back as
`.../141-O`.

This adds `'` and `*` to the path/query class of both regexes (no logic
changes) so those URLs extract intact. `(` and `)` were already allowed
there.

Added two `extractUrls` cases (apostrophe, asterisk) that fail before
the change and pass after.

Author: anxkhn

packages/utils/src/internals/general.ts

@@ -6,14 +6,14 @@ import { setTimeout } from 'node:timers/promises';
  * and does not support URLs containing commas or spaces. The URLs also may contain Unicode letters (not symbols).
  */
 export const URL_NO_COMMAS_REGEX =
-    /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()]*)?/giu;
+    /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+.~#?&/=()'*]*)?/giu;
 
 /**
  * Regular expression that, in addition to the default regular expression `URL_NO_COMMAS_REGEX`, supports matching commas in URL path and query.
  * Note, however, that this may prevent parsing URLs from comma delimited lists, or the URLs may become malformed.
  */
 export const URL_WITH_COMMAS_REGEX =
-    /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()]*)?/giu;
+    /https?:\/\/(www\.)?([\p{L}0-9]|[\p{L}0-9][-\p{L}0-9@:%._+~#=]{0,254}[\p{L}0-9])\.[a-z]{2,63}(:\d{1,5})?(\/[-\p{L}0-9@:%_+,.~#?&/=()'*]*)?/giu;
 
 let isDockerPromiseCache: Promise<boolean> | undefined;
 

test/utils/extract-urls.test.ts

@@ -96,6 +96,19 @@ describe('extractUrls()', () => {
         expect(extracted).toEqual(array);
     });
 
+    // https://github.com/apify/crawlee/issues/2755
+    test('extracts URLs with apostrophes in the path', () => {
+        const url = "https://www.zillow.com/homedetails/141-O'Canoe-Pl-Hertford-NC-27944/74398007_zpid/";
+        const extracted = extractUrls({ string: url });
+        expect(extracted).toEqual([url]);
+    });
+
+    test('extracts URLs with asterisks in the path', () => {
+        const url = 'https://example.com/path*star/end';
+        const extracted = extractUrls({ string: url });
+        expect(extracted).toEqual([url]);
+    });
+
     test('does not extract invalid URLs', () => {
         const { string } = getURLData(INVALID_URL_LIST);
         const extracted = extractUrls({ string });