Treat parser HTTP fallback failures as noresults #685
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Parallel Tests | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| branches: [dev, main, master] | |
| push: | |
| branches: [dev, main, master] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| PYTHONIOENCODING: utf-8 | |
| PYTHONLEGACYWINDOWSSTDIO: utf-8 | |
| USE_COLOR: False | |
| UV_NO_SOURCES: "1" | |
| jobs: | |
| prek: | |
| name: Pre-commit checks | |
| runs-on: ubuntu-24.04 | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 1 | |
| - name: Clone abxpkg | |
| run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git "${{ runner.temp }}/abxpkg" | |
| - name: Set up Python 3.13 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: "3.13" | |
| architecture: x64 | |
| - name: Install uv | |
| run: python -m pip install --upgrade uv | |
| - name: Set up Node JS | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version: 22 | |
| - name: Enable pnpm | |
| run: | | |
| corepack enable | |
| PNPM_HOME="$HOME/.local/share/pnpm" | |
| mkdir -p "$PNPM_HOME" | |
| pnpm config set global-bin-dir "$PNPM_HOME" | |
| echo "PNPM_HOME=$PNPM_HOME" >> "$GITHUB_ENV" | |
| echo "$PNPM_HOME" >> "$GITHUB_PATH" | |
| - name: Install JS tooling | |
| run: pnpm add -g prettier | |
| - name: Cache uv | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/uv | |
| key: ${{ runner.os }}-3.13-uv-${{ hashFiles('pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-3.13-uv- | |
| - name: Install system packages | |
| run: | | |
| sudo apt-get update | |
| sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ | |
| git wget ripgrep build-essential python3-dev python3-setuptools \ | |
| libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 \ | |
| python3-minimal gnupg2 curl python3-ldap python3-msgpack \ | |
| python3-mutagen python3-regex python3-pycryptodome procps | |
| - name: Install dependencies with uv | |
| run: | | |
| uv venv | |
| uv pip install --group dev -e "${{ runner.temp }}/abxpkg" -e . | |
| - name: Run prek | |
| env: | |
| PYTHONPATH: ${{ runner.temp }}/abxpkg | |
| run: uv run --no-sync --no-sources prek run --all-files | |
| changes: | |
| name: Detect changed paths | |
| runs-on: ubuntu-24.04 | |
| outputs: | |
| docs-only: ${{ steps.changed-files.outputs.docs-only }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 2 | |
| - name: Detect docs-only changes | |
| id: changed-files | |
| run: | | |
| docs_only=false | |
| if git rev-parse --verify HEAD^ >/dev/null 2>&1; then | |
| changed_files=$(git diff --name-only HEAD^ HEAD | sed '/^$/d') | |
| if [ -n "$changed_files" ] && ! printf '%s\n' "$changed_files" | grep -qv '^docs/'; then | |
| docs_only=true | |
| fi | |
| fi | |
| echo "docs-only=$docs_only" >> "$GITHUB_OUTPUT" | |
| echo "docs-only=$docs_only" | |
| discover-tests: | |
| name: Discover test files | |
| runs-on: ubuntu-24.04 | |
| needs: [changes] | |
| if: ${{ needs.changes.outputs.docs-only != 'true' }} | |
| outputs: | |
| test-files: ${{ steps.set-matrix.outputs.test-files }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 1 | |
| - name: Discover test files | |
| id: set-matrix | |
| run: | | |
| plugin_tests=$(find abx_plugins/plugins -path "*/tests/test_*.py" -type f | sort) | |
| root_tests="" | |
| if [ -d tests ]; then | |
| root_tests=$(find tests -path "*/test_*.py" -type f | sort) | |
| fi | |
| all_tests=$(printf "%s\n%s\n" "$plugin_tests" "$root_tests" | sed '/^$/d') | |
| json_array="[" | |
| first=true | |
| for test_file in $all_tests; do | |
| if [ "$first" = true ]; then | |
| first=false | |
| else | |
| json_array+="," | |
| fi | |
| test_name=$(basename $test_file .py | sed 's/^test_//') | |
| name="$test_name" | |
| needs_chromium=false | |
| needs_brew=false | |
| if [[ "$test_file" == abx_plugins/plugins/*/tests/* ]]; then | |
| plugin_dir="${test_file%/tests/*}" | |
| if grep -Riq "chrom" "$plugin_dir"; then | |
| needs_chromium=true | |
| fi | |
| elif grep -iq "chrom" "$test_file"; then | |
| needs_chromium=true | |
| fi | |
| if [[ "$test_file" == abx_plugins/plugins/brew/tests/* ]]; then | |
| needs_brew=true | |
| fi | |
| json_array+="{\"path\":\"$test_file\",\"name\":\"$name\",\"needs_chromium\":$needs_chromium,\"needs_brew\":$needs_brew}" | |
| done | |
| json_array+="]" | |
| echo "test-files=$json_array" >> $GITHUB_OUTPUT | |
| echo "Found $(echo $all_tests | wc -w) test files" | |
| echo "$json_array" | jq '.' | |
| run-tests: | |
| name: ${{ matrix.test.name }} | |
| runs-on: ubuntu-24.04 | |
| needs: [prek, changes, discover-tests] | |
| if: ${{ needs.changes.outputs.docs-only != 'true' }} | |
| env: | |
| ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} | |
| ABX_PYTEST_TEE_SUBPROCESS_OUTPUT: "1" | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 20 | |
| matrix: | |
| test: ${{ fromJson(needs.discover-tests.outputs.test-files) }} | |
| python: ["3.13"] | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v5 | |
| with: | |
| fetch-depth: 1 | |
| - name: Clone abxpkg | |
| run: git clone --depth=1 https://github.com/ArchiveBox/abxpkg.git "${{ runner.temp }}/abxpkg" | |
| - name: Set up Python ${{ matrix.python }} | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ matrix.python }} | |
| architecture: x64 | |
| - name: Install uv | |
| run: python -m pip install --upgrade uv | |
| - name: Set up Node JS | |
| uses: actions/setup-node@v5 | |
| with: | |
| node-version: 22 | |
| - name: Cache uv | |
| uses: actions/cache@v4 | |
| with: | |
| path: ~/.cache/uv | |
| key: ${{ runner.os }}-${{ matrix.python }}-uv-${{ hashFiles('pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-${{ matrix.python }}-uv- | |
| - name: Install system packages | |
| run: | | |
| sudo apt-get update | |
| sudo DEBIAN_FRONTEND=noninteractive apt-get install -y \ | |
| git wget ripgrep build-essential python3-dev python3-setuptools \ | |
| libssl-dev libldap2-dev libsasl2-dev zlib1g-dev libatomic1 \ | |
| python3-minimal gnupg2 curl python3-ldap python3-msgpack \ | |
| python3-mutagen python3-regex python3-pycryptodome procps | |
| - name: Install Homebrew | |
| if: ${{ matrix.test.needs_brew }} | |
| env: | |
| NONINTERACTIVE: "1" | |
| run: | | |
| /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" | |
| eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)" | |
| echo "$HOMEBREW_PREFIX/bin" >> $GITHUB_PATH | |
| echo "$HOMEBREW_PREFIX/sbin" >> $GITHUB_PATH | |
| echo "PATH=$PATH" >> $GITHUB_ENV | |
| echo "HOMEBREW_PREFIX=$HOMEBREW_PREFIX" >> $GITHUB_ENV | |
| echo "HOMEBREW_CELLAR=$HOMEBREW_CELLAR" >> $GITHUB_ENV | |
| echo "HOMEBREW_REPOSITORY=$HOMEBREW_REPOSITORY" >> $GITHUB_ENV | |
| echo "MANPATH=$MANPATH" >> $GITHUB_ENV | |
| echo "INFOPATH=$INFOPATH" >> $GITHUB_ENV | |
| echo "XDG_DATA_DIRS=$XDG_DATA_DIRS" >> $GITHUB_ENV | |
| brew --version | |
| - name: Install dependencies with uv | |
| run: | | |
| uv venv | |
| uv pip install --group dev -e "${{ runner.temp }}/abxpkg" -e . | |
| - name: Select Chromium for extension-capable CDP | |
| if: ${{ matrix.test.needs_chromium }} | |
| env: | |
| MIN_CHROMIUM_MAJOR: "149" | |
| LIB_DIR: ${{ runner.temp }}/abx-lib | |
| ABXPKG_LIB_DIR: ${{ runner.temp }}/abx-lib | |
| run: | | |
| set -euo pipefail | |
| mkdir -p "$LIB_DIR" | |
| echo "LIB_DIR=$LIB_DIR" >> "$GITHUB_ENV" | |
| echo "ABXPKG_LIB_DIR=$ABXPKG_LIB_DIR" >> "$GITHUB_ENV" | |
| candidate="/usr/bin/chromium" | |
| version="" | |
| major=0 | |
| if [ -x "$candidate" ]; then | |
| version="$("$candidate" --version || true)" | |
| major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)" | |
| case "$major" in | |
| ''|*[!0-9]*) major=0 ;; | |
| esac | |
| fi | |
| if [ "$major" -ge "$MIN_CHROMIUM_MAJOR" ]; then | |
| echo "Using runner Chromium: $candidate ($version)" | |
| echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV" | |
| exit 0 | |
| fi | |
| echo "Runner Chromium is unavailable or too old: ${version:-not found}" | |
| echo "Installing Playwright Chromium into $LIB_DIR..." | |
| uv run --no-sync --no-sources abxpkg install \ | |
| --binproviders=npm \ | |
| --lib="$LIB_DIR" \ | |
| --postinstall-scripts=false \ | |
| --min-release-age=0 \ | |
| --no-cache \ | |
| --overrides='{"npm":{"install_args":["playwright@next"]}}' \ | |
| playwright | |
| uv run --no-sync --no-sources abxpkg install \ | |
| --binproviders=playwright \ | |
| --lib="$LIB_DIR" \ | |
| --bin-dir="$LIB_DIR/env/bin" \ | |
| --min-version="$MIN_CHROMIUM_MAJOR" \ | |
| chromium | |
| candidate="$LIB_DIR/env/bin/chromium" | |
| if [ ! -x "$candidate" ]; then | |
| echo "Playwright Chromium shim was not created at $candidate" >&2 | |
| exit 1 | |
| fi | |
| version="$("$candidate" --version || true)" | |
| major="$(printf '%s\n' "$version" | sed -E 's/.* ([0-9]+)\..*/\1/' | head -1)" | |
| case "$major" in | |
| ''|*[!0-9]*) major=0 ;; | |
| esac | |
| if [ "$major" -lt "$MIN_CHROMIUM_MAJOR" ]; then | |
| echo "Playwright Chromium is still too old: ${version:-unknown}" >&2 | |
| exit 1 | |
| fi | |
| echo "Using Playwright Chromium: $candidate ($version)" | |
| echo "CHROME_BINARY=$candidate" >> "$GITHUB_ENV" | |
| - name: Verify Chromium launch | |
| if: ${{ matrix.test.needs_chromium }} | |
| uses: ./.github/actions/verify-chromium-launch | |
| - name: Run test - ${{ matrix.test.name }} | |
| env: | |
| PYTHONPATH: ${{ runner.temp }}/abxpkg | |
| TWOCAPTCHA_API_KEY: ${{ secrets.TWOCAPTCHA_API_KEY }} | |
| CHROME_SANDBOX: "false" | |
| CHROME_HEADLESS: "True" | |
| run: | | |
| uv run --no-sync --no-sources pytest -xvs -rA "${{ matrix.test.path }}" --basetemp="$RUNNER_TEMP/pytest-out" |