fix: update handleCloudflareChallenge for new Cloudflare challenge markup (#3717)

## What

`handleCloudflareChallenge` stopped detecting the Cloudflare challenge.
Cloudflare inserted a `.footer-wrapper` element between `.footer-inner`
and `.diagnostic-wrapper` on the challenge page, which broke the
direct-child detection selector — so the helper returned immediately and
the request handler ran against an unsolved "Just a moment..." page
(exactly the report in #3629).

- Detection now matches by **descendants** (`.footer .footer-inner
.diagnostic-wrapper .ray-id`) so intervening wrappers don't break it.
- The click position now targets the **actual challenge widget** (the
box holding the Turnstile checkbox input), falling back to the old
`.main-content div` for older layouts.

## e2e alignment

The `camoufox-cloudflare` e2e was also a false green: its own
`isBlocked` check used the same broken selector, so the assertion always
passed. Fixed that selector, and aligned the browser stack —
`camoufox-js@^0.11.0` installs the Camoufox **150** (Firefox 150) build,
so the actor now uses **Playwright 1.60** (also Firefox 150). A
mismatched Firefox crashes playwright-core on Cloudflare's uncaught page
errors (`pageError.location` is undefined), so the versions must line
up.

Depends on camoufox-js ≥ 0.11.0 (apify/camoufox-js#283).

Closes #3629

Author: B4nan

package.json

@@ -87,7 +87,7 @@
         "apify-node-curl-impersonate": "^1.0.15",
         "basic-auth-parser": "^0.0.2",
         "body-parser": "^2.0.0",
-        "camoufox-js": "^0.10.0",
+        "camoufox-js": "^0.11.0",
         "commitlint": "^21.0.0",
         "cross-env": "^10.0.0",
         "deep-equal": "^2.0.5",

packages/playwright-crawler/src/internals/utils/playwright-utils.ts

@@ -753,7 +753,9 @@ async function handleCloudflareChallenge(
 
     options.isChallengeCallback ??= async () => {
         return await page.evaluate(async () => {
-            return !!document.querySelector('.footer > .footer-inner > .diagnostic-wrapper > .ray-id');
+            // Cloudflare nests the ray ID under varying wrapper elements, so we match by descendants
+            // instead of a direct-child chain (e.g. a `.footer-wrapper` was inserted in between).
+            return !!document.querySelector('.footer .footer-inner .diagnostic-wrapper .ray-id');
         });
     };
 
@@ -782,7 +784,11 @@ async function handleCloudflareChallenge(
 
     const bb = await page
         .evaluate(() => {
-            const div = document.querySelector('.main-content div');
+            // Prefer the actual challenge widget (the box holding the Turnstile checkbox input);
+            // fall back to the first content div for older challenge layouts.
+            const div =
+                document.querySelector('.main-content div:has(input[id^="cf-chl-widget-"])') ??
+                document.querySelector('.main-content div');
             return div?.getBoundingClientRect();
         })
         .catch(() => undefined);

test/e2e/camoufox-cloudflare/actor/main.js

@@ -28,7 +28,7 @@ await Actor.main(async () => {
         async requestHandler({ page, parseWithCheerio }) {
             const isBlocked = await page
                 .evaluate(async () => {
-                    return !!document.querySelector('.footer > .footer-inner > .diagnostic-wrapper > .ray-id');
+                    return !!document.querySelector('.footer .footer-inner .diagnostic-wrapper .ray-id');
                 })
                 .catch(() => false);
             const $ = await parseWithCheerio();

test/e2e/camoufox-cloudflare/actor/package.json

@@ -13,8 +13,8 @@
         "@crawlee/playwright": "file:./packages/playwright-crawler",
         "@crawlee/types": "file:./packages/types",
         "@crawlee/utils": "file:./packages/utils",
-        "camoufox-js": "^0.9.3",
-        "playwright": "1.58.2"
+        "camoufox-js": "^0.11.0",
+        "playwright": "1.60.0"
     },
     "overrides": {
         "apify": {

yarn.lock

@@ -1141,7 +1141,7 @@ __metadata:
     apify-node-curl-impersonate: "npm:^1.0.15"
     basic-auth-parser: "npm:^0.0.2"
     body-parser: "npm:^2.0.0"
-    camoufox-js: "npm:^0.10.0"
+    camoufox-js: "npm:^0.11.0"
     commitlint: "npm:^21.0.0"
     cross-env: "npm:^10.0.0"
     deep-equal: "npm:^2.0.5"
@@ -4401,14 +4401,14 @@ __metadata:
   languageName: node
   linkType: hard
 
-"better-sqlite3@npm:^12.2.0":
-  version: 12.4.1
-  resolution: "better-sqlite3@npm:12.4.1"
+"better-sqlite3@npm:^12.10.0":
+  version: 12.10.0
+  resolution: "better-sqlite3@npm:12.10.0"
   dependencies:
     bindings: "npm:^1.5.0"
     node-gyp: "npm:latest"
     prebuild-install: "npm:^7.1.1"
-  checksum: 10c0/88773a75d996b4171e5690a38459b05dc814a792701b224bd9909ee084dc0b4c64aaffbdbcf4bbbc6d4e247faf19e91b2a56cf4175d746d3bd9ff14764eb05aa
+  checksum: 10c0/b269792c1e456201b16fb9abd1e3fa9284d7710929f991899cebbb61e56dc3703673be29dadcb2f4a2d02961c49145f53d7d7b13b7b47298c033ec9de524f34f
   languageName: node
   linkType: hard
 
@@ -4764,12 +4764,12 @@ __metadata:
   languageName: node
   linkType: hard
 
-"camoufox-js@npm:^0.10.0":
-  version: 0.10.2
-  resolution: "camoufox-js@npm:0.10.2"
+"camoufox-js@npm:^0.11.0":
+  version: 0.11.0
+  resolution: "camoufox-js@npm:0.11.0"
   dependencies:
     adm-zip: "npm:^0.5.16"
-    better-sqlite3: "npm:^12.2.0"
+    better-sqlite3: "npm:^12.10.0"
     cli-progress: "npm:^3.12.0"
     commander: "npm:^14.0.0"
     fingerprint-generator: "npm:^2.1.66"
@@ -4784,7 +4784,7 @@ __metadata:
     playwright-core: "*"
   bin:
     camoufox-js: dist/__main__.js
-  checksum: 10c0/f04cd3b46f507649d7d2c882ad5f9e1a3a4420480d5a196b40d26ca635d2d2193764625fa6518baa17a2283445a689194aa2139c67825f1326604870859f7890
+  checksum: 10c0/ac17d937edd690133638568e34726fe5b60ce763d6252edf163e6326571de7263d6103facd5fb3a7ca6979d7339aad454c588eef945676e96a8083d06c884165
   languageName: node
   linkType: hard