fix: Invoke deferred cleanup after error handling #10387
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Check | |
| on: | |
| push: | |
| branches: [ master, v4, renovate/** ] | |
| pull_request: | |
| branches: [ master, v4 ] | |
| env: | |
| RETRY_TESTS: 1 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| build_and_test: | |
| name: Build & Test | |
| if: (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:')) | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # tests on windows are extremely unstable | |
| # os: [ ubuntu-22.04, windows-2019 ] | |
| os: [ ubuntu-22.04 ] | |
| node-version: [ 22, 24 ] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v7 | |
| - name: Use Node.js ${{ matrix.node-version }} | |
| uses: actions/setup-node@v7 | |
| with: | |
| node-version: ${{ matrix.node-version }} | |
| package-manager-cache: false | |
| - name: Turbo cache | |
| id: turbo-cache | |
| uses: actions/cache@v6 | |
| with: | |
| path: .turbo | |
| key: turbo-${{ github.job }}-${{ matrix.node-version }}-${{ github.ref_name }}-${{ github.sha }} | |
| restore-keys: | | |
| turbo-${{ github.job }}-${{ matrix.node-version }}-${{ github.ref_name }}- | |
| - uses: apify/workflows/pnpm-install@main | |
| - name: Install Playwright browsers | |
| run: pnpm exec playwright install --with-deps | |
| - name: Install Puppeteer Chrome | |
| run: pnpm exec puppeteer browsers install chrome | |
| - name: Build | |
| run: pnpm ci:build | |
| - name: Test TS | |
| run: pnpm tsc-check-tests | |
| - name: Typecheck documentation examples | |
| working-directory: ./docs | |
| run: pnpm typecheck | |
| - name: Tests | |
| run: pnpm test | |
| api_surface: | |
| name: Public API surface | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v6 | |
| - name: Use Node.js 24 | |
| uses: actions/setup-node@v6 | |
| with: | |
| node-version: 24 | |
| package-manager-cache: false | |
| - name: Turbo cache | |
| id: turbo-cache | |
| uses: actions/cache@v5 | |
| with: | |
| path: .turbo | |
| key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} | |
| restore-keys: | | |
| turbo-${{ github.job }}-${{ github.ref_name }}- | |
| - uses: apify/workflows/pnpm-install@main | |
| - name: Build | |
| run: pnpm ci:build | |
| - name: Check public API surface | |
| run: pnpm api:check | |
| docs: | |
| name: Docs build | |
| if: (!contains(github.event.head_commit.message, '[skip ci]') && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/v4') | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout Source code | |
| uses: actions/checkout@v7 | |
| - name: Use Node.js 24 | |
| uses: actions/setup-node@v7 | |
| with: | |
| node-version: 24 | |
| package-manager-cache: false | |
| - name: Turbo cache | |
| id: turbo-cache | |
| uses: actions/cache@v6 | |
| with: | |
| path: .turbo | |
| key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} | |
| restore-keys: | | |
| turbo-${{ github.job }}-${{ github.ref_name }}- | |
| - uses: apify/workflows/pnpm-install@main | |
| - name: Build & deploy docs | |
| run: | | |
| cd website | |
| pnpm build | |
| env: | |
| APIFY_SIGNING_TOKEN: ${{ secrets.APIFY_SIGNING_TOKEN }} | |
| SEGMENT_TOKEN: ${{ secrets.SEGMENT_TOKEN }} | |
| - name: Install Nginx | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y nginx | |
| - name: Start Docusaurus server | |
| run: | | |
| cd website | |
| nohup yarn docusaurus serve --port 3000 --no-open & | |
| sleep 5 | |
| curl -f http://localhost:3000 > /dev/null | |
| - name: Start Nginx with project config | |
| run: | | |
| PWD_PATH="$(pwd)" | |
| cat > default.conf <<EOF | |
| worker_processes auto; | |
| error_log ${PWD_PATH}/logs/error.log; | |
| pid ${PWD_PATH}/logs/nginx.pid; | |
| events {} | |
| http { | |
| access_log ${PWD_PATH}/logs/access.log; | |
| include ${PWD_PATH}/website/nginx.conf; | |
| } | |
| EOF | |
| sed -i 's|https://apify.github.io/crawlee|http://localhost:3000|g' default.conf | |
| mkdir -p "${PWD_PATH}/logs" | |
| nginx -c "${PWD_PATH}/default.conf" | |
| sleep 1 | |
| - name: Run header assertions | |
| run: | | |
| set -euo pipefail | |
| function assert_header() { | |
| url=$1 | |
| header=$2 | |
| expected=$3 | |
| shift 3 | |
| extra_args=("$@") | |
| actual=$(curl -s -D - -o /dev/null "${extra_args[@]}" "$url" | grep -i "^$header" | tr -d '\r' || true) | |
| echo "→ $url → $actual" | |
| echo "$actual" | grep -q "$expected" || (echo "❌ Expected '$expected' in '$header' for $url" && exit 1) | |
| } | |
| function assert_status() { | |
| url=$1 | |
| expected=$2 | |
| shift 2 | |
| extra_args=("$@") | |
| actual=$(curl -s -o /dev/null -w "%{http_code}" "${extra_args[@]}" "$url") | |
| echo "→ $url → HTTP $actual" | |
| [ "$actual" = "$expected" ] || (echo "❌ Expected HTTP $expected but got $actual for $url" && exit 1) | |
| } | |
| function assert_no_redirect() { | |
| url=$1 | |
| shift | |
| extra_args=("$@") | |
| response=$(curl -s -D - -o /dev/null -w "\n%{http_code}" "${extra_args[@]}" "$url" 2>/dev/null) | |
| status=$(echo "$response" | tail -1) | |
| location=$(echo "$response" | grep -i "^location:" | tr -d '\r' || true) | |
| echo "→ $url → HTTP $status ${location:+(${location})}" | |
| if [ "$status" = "301" ] || [ "$status" = "302" ]; then | |
| echo "❌ Got redirect for $url: $location" && exit 1 | |
| fi | |
| } | |
| echo "🧪 Checking open redirect protection..." | |
| assert_no_redirect "http://localhost:8080///%5Cevil.com/" | |
| assert_no_redirect "http://localhost:8080/%5Cevil.com/" | |
| assert_no_redirect "http://localhost:8080///%5cevil.com/" | |
| assert_no_redirect "http://localhost:8080" --request-target '/\evil.com/' | |
| assert_no_redirect "http://localhost:8080" --request-target '///\evil.com/' | |
| assert_status "http://localhost:8080/js/docs/quick-start/" "302" | |
| echo "🧪 Checking Nginx responses... (crawlee JS)" | |
| assert_header "http://localhost:8080/" "Content-Type" "text/html" | |
| assert_header "http://localhost:8080/" "Content-Type" "text/markdown" -H "Accept: text/markdown" | |
| assert_header "http://localhost:8080/js/docs/quick-start" "Content-Type" "text/html" | |
| assert_header "http://localhost:8080/js/docs/quick-start.md" "Content-Type" "text/markdown" | |
| assert_header "http://localhost:8080/js/docs/quick-start" "Content-Type" "text/markdown" -H "Accept: text/markdown" | |
| assert_header "http://localhost:8080/llms.txt" "Content-Type" "text/markdown" | |
| assert_header "http://localhost:8080/llms-full.txt" "Content-Type" "text/markdown" | |
| echo "🧪 Checking Nginx responses... (crawlee Python)" | |
| assert_header "http://localhost:8080/python/docs/quick-start" "Content-Type" "text/html" | |
| assert_header "http://localhost:8080/python/docs/quick-start.md" "Content-Type" "text/markdown" | |
| assert_header "http://localhost:8080/python/docs/quick-start" "Content-Type" "text/markdown" -H "Accept: text/markdown" | |
| assert_header "http://localhost:8080/python/llms.txt" "Content-Type" "text/markdown" | |
| assert_header "http://localhost:8080/python/llms-full.txt" "Content-Type" "text/markdown" | |
| echo "✅ All Nginx header checks passed." | |
| - name: Stop Nginx | |
| if: always() | |
| run: nginx -c "$(pwd)/default.conf" -s stop | |
| lint: | |
| name: Lint | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v7 | |
| - name: Use Node.js 24 | |
| uses: actions/setup-node@v7 | |
| with: | |
| node-version: 24 | |
| package-manager-cache: false | |
| - name: Turbo cache | |
| id: turbo-cache | |
| uses: actions/cache@v6 | |
| with: | |
| path: .turbo | |
| key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} | |
| restore-keys: | | |
| turbo-${{ github.job }}-${{ github.ref_name }}- | |
| - uses: apify/workflows/pnpm-install@main | |
| - name: Oxlint | |
| run: pnpm lint | |
| - name: Oxfmt format check | |
| run: pnpm format:check | |
| release_next: | |
| name: Release @next | |
| if: github.event_name == 'push' && contains(github.event.ref, 'v4') && (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:')) | |
| needs: build_and_test | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v7 | |
| with: | |
| token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} | |
| fetch-depth: 0 | |
| - name: Use Node.js 24 | |
| uses: actions/setup-node@v7 | |
| with: | |
| node-version: 24 | |
| package-manager-cache: false | |
| - name: Turbo cache | |
| id: turbo-cache | |
| uses: actions/cache@v6 | |
| with: | |
| path: .turbo | |
| key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }} | |
| restore-keys: | | |
| turbo-${{ github.job }}-${{ github.ref_name }}- | |
| - uses: apify/workflows/pnpm-install@main | |
| - name: Build | |
| run: pnpm ci:build | |
| - name: Generate changed packages list | |
| id: changed-packages | |
| # `set -eo pipefail` plus an explicit assignment ensures a lerna | |
| # crash fails the step instead of being swallowed by the outer | |
| # `echo | tee` (which would otherwise emit `changed_packages=0` | |
| # and skip the publish step silently). | |
| run: | | |
| set -eo pipefail | |
| changed=$(node ./node_modules/.bin/lerna changed -p | wc -l | xargs) | |
| echo "changed_packages=$changed" | tee -a "$GITHUB_OUTPUT" | |
| - name: Report nothing to release | |
| if: steps.changed-packages.outputs.changed_packages == '0' | |
| run: echo "Nothing to release" | |
| - name: Publish packages | |
| if: steps.changed-packages.outputs.changed_packages != '0' | |
| uses: apify/actions/execute-workflow@v1.4.0 | |
| with: | |
| workflow: publish-to-npm.yml | |
| inputs: > | |
| { | |
| "ref": "${{ steps.commit.outputs.commit_long_sha || github.sha }}", | |
| "dist-tag": "v4" | |
| } | |
| # Docker image builds are only published off master; gated here so this block | |
| # carries cleanly to master when v4 is integrated, without firing on v4 pushes. | |
| - name: Collect versions for Docker images | |
| id: versions | |
| if: github.ref == 'refs/heads/master' | |
| run: | | |
| crawlee=`node -p "require('./packages/crawlee/package.json').version"` | |
| echo "crawlee=$crawlee" | tee -a $GITHUB_OUTPUT | |
| - name: Trigger Docker image builds | |
| uses: peter-evans/repository-dispatch@v4 | |
| # Trigger next images only if we have something new pushed | |
| if: github.ref == 'refs/heads/master' && steps.changed-packages.outputs.changed_packages != '0' | |
| with: | |
| token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} | |
| repository: apify/apify-actor-docker | |
| event-type: build-node-images | |
| client-payload: > | |
| { | |
| "crawlee_version": "${{ steps.versions.outputs.crawlee }}", | |
| "release_tag": "beta" | |
| } |