pr-validation.yml

name: PR Validation

on:
  pull_request:
    branches: [main]

jobs:
  typecheck:
    name: TypeScript Check
    runs-on: ubuntu-latest
    timeout-minutes: 5

    steps:
      - uses: actions/checkout@v7

      - uses: actions/setup-node@v7
        with:
          node-version: "20"
          cache: "npm"

      - name: Install dependencies
        run: npm ci

      - name: TypeScript check
        run: npm run typecheck

      # Unit tests for the dependency-free helpers under scripts/ — plain
      # `node --test`, no Playwright and no backend, so they belong in this fast
      # job rather than the E2E one. These cover CI-critical logic that had no
      # test job at all before: the duration-balanced shard partitioner (#936)
      # and the report-integrity guard (#1012). A guard whose own test never
      # runs is a guard nobody is guarding.
      - name: Script unit tests
        run: npm run test:scripts

      # The TypeScript half of the same idea (#1017). Everything written in `.ts`
      # was ungated: the `@stable` parser two generators read as the release
      # signal (#985), the script that EDITS SPECS AND COMMITS TO `main` with no
      # human review, and the collect-models fallback whose scenario matrix was
      # validated in a scratch file outside the repo (#1011).
      #
      # Same runner as the lane above — `node --test` — via ts-node's require
      # hook, so no new dependency and no second runner to keep current. The hook
      # (not the deprecated `--loader ts-node/esm`) is what makes this work on the
      # Node 20 this workflow pins; the trade-off and the rejected options are
      # recorded in CONTRIBUTING.md → "Unit tests".
      - name: TypeScript unit tests
        run: npm run test:units

      # The REGRESSIONS.md indicator block is generated and committed by hand.
      # Fail the PR when a row was added without regenerating it, so the
      # headline number can never silently disagree with the table.
      - name: Regression Ledger indicator in sync
        run: npm run regressions:check

  lint:
    name: ESLint
    runs-on: ubuntu-latest
    timeout-minutes: 5

    steps:
      - uses: actions/checkout@v7

      - uses: actions/setup-node@v7
        with:
          node-version: "20"
          cache: "npm"

      - name: Install dependencies
        run: npm ci

      - name: Lint
        run: npm run lint:ci

  # The repo's own Python code had no lane at all, which is how `generate_report.py`
  # shipped a report that said `Result: PASSED` on a run whose UI phase had failed
  # (#1120, run #115). The two `test:units` / `test:scripts` lanes only cover
  # TypeScript and `.mjs`, so nothing here was verifiable before merge.
  python-units:
    name: Python unit tests
    runs-on: ubuntu-latest
    timeout-minutes: 5

    steps:
      - uses: actions/checkout@v7

      - uses: actions/setup-python@v7
        with:
          python-version: "3.12"

      - name: Install pytest
        run: pip install --quiet pytest

      - name: Migration helper unit tests
        run: |
          # `test_ui_migration.py` is excluded, not missing: it drives a real browser
          # against a migrated Langflow and belongs to migration-test.yml. Everything
          # else under this path is a plain unit test, so a new one is picked up
          # without touching this workflow.
          #
          # Fail loudly on an empty selection, like the two Node lanes do — a unit
          # lane that silently runs nothing is worse than no lane (#1012's rule).
          COUNT=$(python -m pytest tests/github-workflows/migration \
            --ignore=tests/github-workflows/migration/test_ui_migration.py \
            --collect-only -q 2>/dev/null | grep -cE '::' || true)
          if [ "${COUNT:-0}" -eq 0 ]; then
            echo "::error::no Python unit tests were collected — this lane would have passed without verifying anything."
            exit 1
          fi
          echo "collected $COUNT Python unit test(s)"
          python -m pytest tests/github-workflows/migration \
            --ignore=tests/github-workflows/migration/test_ui_migration.py -v

  checklist-guard:
    name: QA-CHECKLIST guard
    runs-on: ubuntu-latest
    timeout-minutes: 6
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0

      - uses: actions/setup-node@v7
        with:
          node-version: "20"
          cache: "npm"

      # Fail the PR if it edits the AUTO-GENERATED blocks of QA-CHECKLIST.md
      # (Coverage Summary table + note, Phase 0 list, Phase 1/2 tables). Those
      # regenerate on merge via update-coverage-summary.yml; PRs must edit only
      # the manual Part II bullets, so concurrent PRs don't collide on the
      # generated count lines (issue #741). No npm install needed — pure git+node,
      # so it runs before the install step and reports fast.
      - name: Guard QA-CHECKLIST.md generated blocks
        env:
          BASE_REF: ${{ github.base_ref }}
        run: |
          git fetch origin "$BASE_REF" --quiet
          node scripts/check-checklist-guard.mjs "origin/$BASE_REF"

      # The coverage guard below parses `@stable` out of the spec ASTs, so it
      # needs ts-node + typescript. `--ignore-scripts` skips Playwright's
      # browser download — nothing here drives a browser.
      - name: Install dependencies
        run: npm ci --ignore-scripts

      # Fail the PR when a spec CLAIMS coverage (carries `@stable`, or has a spec
      # doc under docs/) but no manual Part II bullet references it — such a spec
      # is invisible in every generated count, including the @stable lane the
      # daily workflow reads as release signal (issue #985). Whole-repo check, not
      # diff-based: main is kept clean, so any new drift fails on the PR that
      # introduces it.
      - name: Guard spec ↔ doc ↔ QA-CHECKLIST triad
        run: npm run check:checklist-coverage

  doc-deps-guard:
    name: Spec-doc dependency paths
    runs-on: ubuntu-latest
    timeout-minutes: 6
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0

      - uses: actions/setup-node@v7
        with:
          node-version: "20"

      # The docs this PR touched. Severity turns on this list: a path in a doc the
      # PR changed FAILS, a pre-existing one is reported (#980's coverage-first
      # trade — one upstream rename must not redden every PR that edits an
      # unrelated doc). An empty list is normal and means nothing can fail here.
      - name: Collect the docs this PR changed
        env:
          BASE_REF: ${{ github.base_ref }}
        run: |
          set -eo pipefail
          git fetch origin "$BASE_REF" --quiet
          git diff --name-only --diff-filter=d "origin/$BASE_REF...HEAD" -- docs README.md > all-changed.txt
          # `grep` exits 1 on no match, which under `pipefail` would abort a PR that
          # simply touches no doc — the empty list is a valid, common outcome.
          grep -E '\.md$' all-changed.txt > changed-docs.txt || :
          echo "changed docs: $(wc -l < changed-docs.txt)"
          cat changed-docs.txt

      # Trees only, no blobs, no working tree: 520 KB and ~1.6 s measured, against
      # ~117 MB to materialise the files. `--mode=check-docs` resolves through
      # `git ls-tree`, which is what makes that possible (issue #1298).
      - name: Clone the upstream tree
        run: |
          set -eo pipefail
          for attempt in 1 2 3; do
            if git clone --filter=blob:none --depth 1 --no-checkout \
                 https://github.com/langflow-ai/langflow.git langflow-upstream; then
              exit 0
            fi
            echo "::warning::upstream clone attempt $attempt failed; retrying"
            rm -rf langflow-upstream
            sleep 5
          done
          echo "::error::could not clone langflow-ai/langflow, so no dependency-path verdict exists for this PR. Not treating that as 'every path resolves'."
          exit 1

      # Fails only on the changed docs; the repo-wide sweep is emitted as
      # ::warning:: so drift that lands upstream between PRs stays visible
      # (#1012 — an unevaluated path is unknown, not clean).
      - name: Resolve every External-dependencies path against upstream
        run: |
          set -eo pipefail
          node scripts/watch-upstream-areas.mjs \
            --mode=check-docs \
            --root langflow-upstream \
            --ref origin/main \
            --changed changed-docs.txt | tee -a "$GITHUB_STEP_SUMMARY"

  detect-specs:
    name: Detect changed specs
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      specs: ${{ steps.diff.outputs.specs }}
      has_specs: ${{ steps.diff.outputs.has_specs }}
      needs_models: ${{ steps.diff.outputs.needs_models }}
      full_suite: ${{ steps.diff.outputs.full_suite }}
      impacted_total: ${{ steps.diff.outputs.impacted_total }}
      dropped_count: ${{ steps.diff.outputs.dropped_count }}
      # True when no spec imports the diff but this lane runs what changed, so the
      # lane runs a fixed canary set instead of skipping (#1159).
      canary: ${{ steps.diff.outputs.canary }}
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0

      - id: diff
        name: Resolve the specs this PR impacts
        env:
          BASE_REF: ${{ github.base_ref }}
          # A helper change can reach half the suite (112 specs for #1052's
          # diff), which is a daily-sized run on a PR. Bound it — and NEVER
          # silently: the dropped set is printed and summarised (#1012's
          # silent-cap rule). The E2E job's own budget is 60 min.
          IMPACTED_SPEC_CAP: "20"
        run: |
          git fetch origin "$BASE_REF" --quiet
          # Selection is by IMPORT GRAPH, not by changed-spec glob (#1054).
          # Selecting only changed `*.spec.ts` let the highest-reach change the
          # repo can receive pass with zero specs executed: PR #1052 changed a
          # helper reached by 112 specs and PR #1088 one imported by 135 — both
          # reported "skipping". The resolver walks importers TRANSITIVELY, so a
          # helper behind a Page Object still selects the specs at the end of
          # the chain (a direct-importers-only pass would have found 62 of
          # #1052's 112 and read as coverage). A changed spec selects itself, so
          # the old behaviour is a subset of this one — including
          # `tests/collect-models.spec.ts`, the file directly under `tests/`
          # that the `:(glob)` pathspec existed to catch (#1016/#1015/#1007).
          git diff --name-only --diff-filter=d "origin/$BASE_REF"...HEAD > /tmp/changed.txt
          echo "Changed files:"; sed 's/^/  /' /tmp/changed.txt
          node scripts/impacted-specs-by-import.mjs --stdin --format=json --cap "$IMPACTED_SPEC_CAP" \
            < /tmp/changed.txt > /tmp/impacted.json || { echo "::error::impacted-spec resolution failed"; exit 1; }

          SPECS=$(jq -r '.selected | join(" ")' /tmp/impacted.json)
          TOTAL=$(jq -r '.specs | length' /tmp/impacted.json)
          DROPPED=$(jq -r '.dropped | length' /tmp/impacted.json)
          FULL_SUITE=$(jq -r '.fullSuite' /tmp/impacted.json)
          DIRECT=$(jq -r '.direct | length' /tmp/impacted.json)
          # `.stableSelected` is deliberately NOT read here. It is scoped to `selected`
          # (post-cap), so pairing it with `$TOTAL` (pre-cap) misreported it on every
          # capped run, and the exclusion below can shorten the set again. The `@stable`
          # tally now comes from the verdict's `.stableRun`, over the surviving list
          # (#1226).

          # Resolution only. The old tail — "running $TOTAL minus $DROPPED dropped" — was
          # the same arithmetic #1226 disproved, and it stayed wrong here after the
          # summary was fixed: on a canary it logged "running 0 minus 0" while three
          # specs ran. What runs is logged once, below, after the verdict settles it.
          echo "Impacted specs resolved: $TOTAL ($DIRECT direct, $((TOTAL - DIRECT)) transitive); $DROPPED dropped by the cap"

          # A CI-ONLY diff imports into no spec, so the resolver above honestly
          # returns zero and this lane used to skip — proving the change PARSES,
          # never that it RUNS (#1159). PR #1157 rewired four workflows onto a new
          # composite action with every check green and the E2E lane `skipping`;
          # the action had executed nowhere at merge time, and a bad `uses:` path
          # would have surfaced first as the next daily failing in a step unrelated
          # to any spec.
          #
          # `ci-change-coverage.mjs` answers the other half: does the PR LANE run
          # what changed? It derives that from the YAML (a workflow's `scripts/x`
          # refs and its `uses: ./.github/actions/y`, plus each action's own refs),
          # so a new action wired in here is covered the day it lands.
          #   canary   → the PR lane's own wiring changed: run a 3-spec set so the
          #              lane boots Langflow and walks pre-flight → health gate →
          #              Playwright for real
          #   dispatch → the surface belongs to another lane; name the workflows to
          #              dispatch instead of implying coverage
          CANARY=false
          # Empty unless the classifier ran, so the renderer is only handed a file that
          # exists — `${CI_COVERAGE:+…}` then omits the flag entirely rather than passing
          # a path to nothing.
          CI_COVERAGE=""
          if [ "$TOTAL" -eq 0 ]; then
            node scripts/ci-change-coverage.mjs --stdin --format=json < /tmp/changed.txt > /tmp/ci-coverage.json \
              || { echo "::error::CI-change classification failed — treating as undecidable rather than as 'no CI change'."; exit 1; }
            CI_COVERAGE=/tmp/ci-coverage.json
            VERDICT=$(jq -r .verdict /tmp/ci-coverage.json)
            jq -r '.reasons[] | "  " + .' /tmp/ci-coverage.json
            case "$VERDICT" in
              canary)
                CANARY=true
                SPECS=$(jq -r '.canarySpecs | join(" ")' /tmp/ci-coverage.json)
                echo "::warning::CI-only change to a surface THIS lane runs ($(jq -r '.ciFiles | join(", ")' /tmp/ci-coverage.json)). No spec imports it, so the lane runs the canary instead of skipping (#1159)."
                ;;
              dispatch)
                echo "::warning::CI-only change to $(jq -r '.ciFiles | join(", ")' /tmp/ci-coverage.json), which THIS lane does not run — nothing here proves it works. Dispatch $(jq -r '.dispatchWorkflows | join(", ")' /tmp/ci-coverage.json) on this branch before merging (#1159)."
                ;;
            esac
          fi

          echo "Running: ${SPECS:-<none>}"

          # The SUMMARY is not written here. It is rendered in one pass at the end of
          # this step by `scripts/render-impacted-summary.mjs`, because the run count is
          # only final after the provider verdict below and because everything that
          # qualifies it must print AFTER it, not above it (#1226). The `::warning::`
          # annotations stay inline: they are log-level, unaffected by ordering, and
          # they are what keeps a shortened run from reading as a clean one even if the
          # step dies before it can render (#1012).
          if [ "$FULL_SUITE" = "true" ]; then
            echo "::warning::This PR changes a suite-wide surface (fixtures / playwright.config / global hooks): ALL $TOTAL specs are impacted. PR CI runs a bounded subset — dispatch manual.yml on this branch for the full suite before merging."
          fi
          if [ "$DROPPED" -gt 0 ]; then
            echo "::warning::Capped at $IMPACTED_SPEC_CAP of $TOTAL impacted specs — $DROPPED not run in this PR (listed in the step log)."
          fi

          # `has_specs` and `specs` are emitted at the END of this step, after the
          # provider-coverage verdict below may have removed specs from the run
          # list. Emitting them here as well would write the key twice and leave
          # which value wins up to the runner (#1216).
          {
            echo "full_suite=$FULL_SUITE"
            echo "impacted_total=$TOTAL"
            echo "dropped_count=$DROPPED"
          } >> "$GITHUB_OUTPUT"

          # Decide (a) whether the impacted specs need the Collect models sweep and
          # (b) whether any of them would otherwise run WITHOUT a provider they
          # need. The second question used to resolve silently to "run it anyway",
          # which is how PR #1152 — a one-helper change — produced a red in
          # `agent-component-regression.spec.ts` (`value-dropdown-model_model` never
          # renders with no provider configured) that had nothing to do with its
          # diff. That spec is `@stable` and green in the daily, where the sweep runs.
          #
          # The verdict turns on what the PR is ABOUT, because forcing the sweep for
          # every provider-dependent spec would re-couple unrelated PRs to provider
          # key health (`Collect models` is a HARD gate here) — the #915/#910/#911
          # cost this gate exists to avoid:
          #   - provider-dependent AND changed directly ⇒ sweep, and run it;
          #   - provider-dependent only transitively     ⇒ EXCLUDE it, and say so.
          # A spec that CONSUMES the sweep's output still forces it either way; that
          # asymmetry is deliberate and argued in the script's header.
          #
          # A canary still forces the sweep (#1159), for the health gate below.
          # Logic and wording live in the script so `npm run test:scripts` covers
          # them; a verdict it cannot produce exits 2 and fails this step rather
          # than degrading to "LLM-free" (#1012).
          CANARY_FLAG=""
          if [ "$CANARY" = "true" ]; then CANARY_FLAG="--canary"; fi
          node scripts/provider-dependent-specs.mjs --stdin --format=json \
            --changed-file=/tmp/changed.txt $CANARY_FLAG \
            < /tmp/impacted.json > /tmp/provider.json \
            || {
              echo "::error::provider-coverage verdict failed"
              # Render what CAN still be established (resolution, suite-wide, the cap)
              # and name the gap, so an aborted verdict leaves an explained summary
              # instead of an empty one. `|| true`: this path is already failing, and a
              # renderer error must not replace the real cause (#1226).
              rm -f /tmp/provider.json
              node scripts/render-impacted-summary.mjs --impacted=/tmp/impacted.json \
                --specs="$SPECS" --cap="$IMPACTED_SPEC_CAP" $CANARY_FLAG \
                >> "$GITHUB_STEP_SUMMARY" || true
              exit 1
            }
          NEEDS_MODELS=$(jq -r '.needsModels' /tmp/provider.json)

          # Re-derive the run list from the verdict, so an excluded spec cannot reach
          # Playwright — but NEVER on a canary run. A canary's specs come from
          # `ci-change-coverage.mjs`, not from the import graph, so `impacted.selected`
          # is empty there and overwriting `$SPECS` with the verdict's (empty) run list
          # would blank it, trip `has_specs=false`, and skip the whole E2E job — taking
          # `Collect models` and the health gate with it, since both live in that job.
          # That is precisely the "green check, nothing ran" hole #1159 exists to close.
          if [ "$CANARY" != "true" ]; then
            SPECS=$(jq -r '.run | join(" ")' /tmp/provider.json)
          fi

          # Log-level annotations only; the summary bullets for these are the renderer's
          # job. Kept inline because a `::warning::` is what still tells a reviewer the
          # run was shortened even if this step dies before rendering (#1012).
          if [ "$(jq -r '.excluded | length' /tmp/provider.json)" -gt 0 ]; then
            echo "::warning::$(jq -r '.warning' /tmp/provider.json)"
          fi
          if [ "$(jq -r '.forcedToAvoidEmptyRun' /tmp/provider.json)" = "true" ]; then
            echo "::warning::Every impacted spec needs a provider, so the sweep runs rather than leaving this lane with nothing to execute — decoupling from key health is not worth ALL of a PR's coverage (#1226)."
          fi

          echo "Needs provider models: $NEEDS_MODELS"
          echo "Running after the provider verdict: ${SPECS:-<none>}"

          # The whole summary, rendered once, from the run list this step is about to
          # export (#1226). Nothing about WHAT it says is decided here: the counts, the
          # order, and every caveat live in `render-impacted-summary.mjs`, where they are
          # asserted on OUTPUT by `npm run test:scripts`. Three wrong figures shipped
          # from this block while it was inline shell, each one guarded only by a regex
          # over this file — and each of those regexes was then shown to miss its own
          # mutation. Same reason the verdict above is a script.
          node scripts/render-impacted-summary.mjs \
            --impacted=/tmp/impacted.json --provider=/tmp/provider.json \
            --specs="$SPECS" --cap="$IMPACTED_SPEC_CAP" $CANARY_FLAG \
            ${CI_COVERAGE:+--ci-coverage=$CI_COVERAGE} \
            >> "$GITHUB_STEP_SUMMARY" \
            || { echo "::error::could not render the run summary"; exit 1; }

          # The run list is whatever survived the verdict — emitted here and only
          # here. An empty list skips the E2E job through `has_specs`, which is the
          # only safe outcome: `npx playwright test` with no paths runs the whole
          # suite. The ::warning:: above is what keeps that skip from reading as a
          # pass (#1012).
          HAS_SPECS=false
          if [ -n "$SPECS" ]; then HAS_SPECS=true; fi
          {
            echo "has_specs=$HAS_SPECS"
            echo "specs=$SPECS"
            echo "needs_models=$NEEDS_MODELS"
            echo "canary=$CANARY"
          } >> "$GITHUB_OUTPUT"

  e2e:
    name: Run impacted E2E specs
    needs: detect-specs
    if: needs.detect-specs.outputs.has_specs == 'true' && github.event.pull_request.head.repo.full_name == github.repository
    runs-on: ubuntu-latest
    timeout-minutes: 60

    services:
      langflow:
        image: langflowai/langflow-nightly:latest
        ports:
          - 7860:7860
        env:
          LANGFLOW_AUTO_LOGIN: "true"
          LANGFLOW_SUPERUSER: langflow
          LANGFLOW_SUPERUSER_PASSWORD: langflow123
          # Cap the backend to one worker. Langflow's image default is (2*cpu)+1
          # workers, each holding full in-memory state; under the collect-models
          # load on the runner they contend until requests hang, and the
          # impacted-specs preflight then times out on /api/v1/version after
          # collect-models succeeds (#922, blocked #867). The launch scripts
          # already default LANGFLOW_WORKERS=1 for this reason (#773) — the
          # service container was the one place missing it.
          LANGFLOW_WORKERS: "1"
          # Cap how long ONE wedge can cost (#1048). Default is 300 s
          # (`worker_timeout` in lfx runtime settings → gunicorn's `timeout`).
          # LangflowUvicornWorker is ASYNC, so this is a heartbeat watchdog on the
          # event loop, not a request deadline — build duration cannot trip it, and
          # Langflow's published docs describe it wrongly. Mechanism, the docs
          # caveat, and the startup measurement: see daily-stable.yml.
          #
          # This job is where the cost was measured. Run 30410211167 (PR #1042)
          # burned two full 300 s outages: the post-collect-models wedge the health
          # gate below had to wait out (227 s, cleared only by gunicorn's kill),
          # and a second block starting within seconds of a failing Google call.
          # Four worker kill/restart cycles in 26 min, and the collateral landed on
          # specs that make no LLM call at all (general-bugs-shard-3909 timing out
          # on `add-project-button`). A wedged worker never self-heals, so waiting
          # 300 s buys nothing; at 120 the kill lands 60-120 s after the stall
          # instead of 150-300 s.
          #
          # This BOUNDS the damage, it does not prevent it: a spec whose API call
          # has a 20 s timeout still dies inside that window. Eliminating
          # collateral needs the provider-heavy specs not to share the backend with
          # unrelated ones (serialization / low-concurrency lane) — see #1048.
          # Rollback is this one value.
          LANGFLOW_WORKER_TIMEOUT: "120"
          # Nightly image defaults LANGFLOW_ALLOW_CUSTOM_COMPONENTS=false
          # (custom-component creation disabled → sidebar button hidden, API
          # 403). Enable it so the custom-component specs exercise the feature.
          LANGFLOW_ALLOW_CUSTOM_COMPONENTS: "true"
          # A2A is off by default (lfx `a2a_enabled=False`) and its router is ALWAYS
          # mounted, so with the flag off the three /api/v1/a2a/* routes answer 404 —
          # indistinguishable from "not mounted". Enable it so the
          # core-functionality/a2a specs exercise the surface instead of asserting
          # against a disabled one, which would pass while testing nothing (#1240;
          # surface scoped in #1195).
          LANGFLOW_A2A_ENABLED: "true"
          # Tracing is OFF by default (cuts startup time / backend noise for the
          # majority of specs). It is turned ON for two independent reasons, and
          # the second one is why this is no longer a single condition (#1300):
          #
          #   1. The observability/traces specs need the SUT to emit traces +
          #      spans, and without it their setup times out.
          #   2. Langflow's traces are the ONLY place this lane's token spend can
          #      be read (the recorder below polls /api/v1/monitor/traces, #1197),
          #      and a flow's trace 404s the moment the flow is deleted.
          #
          # Reason 2 was missing, and it made #1210's recorder on this lane
          # structurally unable to see anything: EVERY run that produced a token
          # artifact after #1210 merged recorded `0 trace(s)` — all 35 of them,
          # 30855127426 through 31021593309, each carrying a `token-attrib.jsonl`
          # and no `token-probes.jsonl` — because tracing was off on every one.
          # Run 31018914069 is the clean measurement: 19 `llm-agents` specs plus
          # collect-models, a provider configured (`needs_models: true`,
          # `Collect models` ran), 59 passed / 29 skipped, and an artifact
          # carrying 139 attribution COST records and not one trace. The
          # summarizer's own honest-zero message names this flag first among the
          # candidate causes, and it was right.
          #
          # `needs_models` is the lane's own verdict for "this run will execute at
          # least one provider-dependent spec" (scripts/provider-dependent-specs.mjs,
          # #1216) — the closest thing the lane has to "this run has spend to
          # measure", and not the same statement. Two known gaps, neither worth a
          # second mechanism: `@playground` is deliberately not a provider tag
          # there, and the provider keys are in the Playwright step's env on every
          # run, so a spec that installs its own credential and pins a model would
          # spend without setting the verdict. Gating on it rather than switching
          # tracing on unconditionally keeps the cheap path for the import-graph
          # majority of PRs, while the daily and manual, which measure every run,
          # keep tracing on unconditionally (daily-stable.yml has been hardcoded
          # `"false"` since #459, before the token monitor existed — the previous
          # comment here misattributed that to #1197).
          #
          # What enabling it COSTS. The "cuts startup time / backend noise"
          # rationale above is what is being traded away, so be exact about what
          # is and is not known.
          #
          # NOT measured: the startup time, and the backend noise. Nothing in
          # #1300 measures either, and this comment does not claim to.
          #
          # Measured: daily-stable.yml has run with tracing on unconditionally
          # since #459 (2026-06-30), and the setting has never been touched since
          # (`git log -S`/`-G`, one commit), across 65 runs of the full `@stable`
          # sweep — 26 scheduled, 39 dispatched, 2 of them cancelled. So an LLM
          # run on this lane now boots in a configuration with ~5 weeks and 26
          # scheduled runs behind it, not a novel one. That is an argument from
          # precedent. It is weaker than a measurement, and it is what exists.
          #
          # One attribution NOT to make, because an earlier version of this
          # comment made it and this file refutes it 60 lines above: the wedge is
          # not wholly a `Collect models` problem. CLAUDE.md does attribute
          # #922/#927 to that sweep, but the LANGFLOW_WORKER_TIMEOUT note above
          # records run 30410211167 burning TWO 300 s outages — the
          # post-collect-models wedge, AND "a second block starting within seconds
          # of a failing Google call", i.e. during the spec run itself. That
          # second class is exactly the class of run this change newly traces. It
          # is bounded by the 120 s worker timeout, not eliminated, and tracing
          # has not been shown to affect it either way.
          #
          # One class does NOT keep the cheap path, and it is the one to know
          # about: decideProviderCoverage() sets `needs_models=true` for a CANARY
          # run (a CI-only diff, three LLM-free specs), so those now boot with
          # tracing on and record another honest zero. That follows from reusing
          # the lane's own verdict instead of inventing a second one, and the
          # cost is a traced boot on a class of PR that is already paying for a
          # full Langflow + `Collect models` sweep.
          LANGFLOW_DEACTIVATE_TRACING: ${{ (contains(needs.detect-specs.outputs.specs, 'observability-monitoring') || needs.detect-specs.outputs.needs_models == 'true') && 'false' || 'true' }}
          # Let Langflow reach the sibling go-httpbin service below (#1128). Its
          # SSRF layer blocks private addresses unless pre-authorized, and the
          # API Request component's validators.url() rejects a single-label host
          # — so ECHO_BASE_URL is a raw container IP and the RFC-1918 ranges are
          # authorized by CIDR rather than by a fixed address the Docker network
          # assigns per run. Same set as daily-stable.yml (#462).
          LANGFLOW_SSRF_ALLOWED_HOSTS: "172.16.0.0/12,10.0.0.0/8,192.168.0.0/16"
        options: >-
          --health-cmd "curl -f http://localhost:7860/health_check || exit 1"
          --health-interval 15s
          --health-timeout 10s
          --health-retries 10
          --health-start-period 90s
      # Self-hosted echo endpoint for the API Request and agent-fetch-tool specs.
      # Without it this lane calls public httpbin.org / postman-echo.com and reds
      # on their outages: PR #1133 lost three specs to an httpbin 504 while its
      # own diff touched none of them, and the log read like a product failure
      # (#1128; earlier recurrences #383/#407/#462/#639).
      #
      # This job runs on the RUNNER HOST, not in a container, so the service
      # alias is not resolvable here — `resolve-echo-endpoint` reads the
      # container IP with `docker inspect` instead, probes the published port,
      # and hands Langflow the IP. Tag has NO `v` prefix (#639).
      # No healthcheck: the scratch-based image ships no shell or curl, so the
      # action's poll is the healthcheck.
      go-httpbin:
        image: ghcr.io/mccutchen/go-httpbin:2.23.1
        ports:
          - 8080:8080

    # The token sidecar's bounds, at JOB level because TWO steps read them: the
    # poller below and the Playwright step, where the attribution sidecar runs
    # inside `deleteFlow`'s teardown hook. They used to live in the poller step's
    # own `env:`, which does not cross into another step -- so the sidecar never
    # saw either one and silently used its hard-coded defaults. The numbers
    # happened to match, so nothing was wrong in the run and nothing could be seen
    # in a diff; the only symptom was a knob that did not turn, precisely when a
    # wedged monitor endpoint made someone want to turn it.
    #
    # Defined ONCE, here, so the poller and the sidecar cannot end up bounded by
    # different numbers. Pinned by scripts/token-sidecar-knobs.test.mjs.
    env:
      TOKENS_TIMEOUT_MS: "8000"
      TOKENS_DETAIL_CAP: "25"
      # Per-CALL wall-clock ceiling for the attribution sidecar. Wired AHEAD of its
      # reader on purpose: the budget itself lands with #1217's `deleteFlow` hook,
      # and adding it from that PR would mean touching THIS file from a spec change,
      # which flips this lane's coverage verdict to `canary` and drops its
      # impacted-spec run. An unread variable costs nothing; a lane that cannot turn
      # the knob when a monitor endpoint wedges costs a run. With the two above it
      # bounds a call at budget + timeout = 23s instead of cap x timeout = 208s.
      # Read by the sidecar only; the poller has its own TOKENS_MAX_SECONDS.
      TOKENS_BUDGET_MS: "15000"

    steps:
      - uses: actions/checkout@v7

      - uses: actions/setup-node@v7
        with:
          node-version: "20"
          cache: "npm"

      - run: npm ci

      - name: Install Playwright browsers
        uses: ./.github/actions/setup-playwright

      # Before ANY spec runs, including collect-models: point the echo-dependent
      # specs at the sibling go-httpbin service instead of the public internet
      # (#1128). `mode: fail` is deliberate and is the one place this lane differs
      # from the daily: a PR is the lane a human is waiting on, so a silent
      # fallback to public httpbin.org — which is what reded PR #1133 and read as
      # a product failure — is worse than a named infra failure it can be
      # re-dispatched from. The daily keeps `warn`, where a day of coverage
      # outweighs strictness.
      - name: Resolve echo endpoint
        uses: ./.github/actions/resolve-echo-endpoint
        with:
          mode: fail
          in_container: "false"

      # Collect provider/model data BEFORE the impacted specs, mirroring
      # daily-stable.yml. Without this, `models.json` is absent (it is
      # git-ignored) and an agent/LLM spec's `resolveTestTargets` falls back to
      # picking a model straight from the live agent dropdown by name — without
      # validating that the key can access it. On some nightly builds that
      # resolves to a model the CI OPENAI_API_KEY's project cannot use (e.g.
      # `gpt-4.1-nano`), so the Agent build 403s and the spec dies on setup,
      # unrelated to the PR (#873). `collect-models` probes each key with a real
      # ~1-token completion and writes only ACCESSIBLE models, so agent specs
      # resolve a usable model — matching what the daily validates (no
      # divergence). Same provider keys as daily-stable.yml.
      # Skipped when no impacted spec needs provider model data (see detect-specs
      # → needs_models). Keeps a provider billing/quota outage from reddening a
      # PR whose specs never use that provider (#915/#910/#911 class).
      - name: Collect models
        if: needs.detect-specs.outputs.needs_models == 'true'
        # Fatal on a normal LLM PR (a spec that needs models must not run without
        # them), NON-fatal on a canary run: there the sweep exists only so the
        # health gate below has something to gate, and nothing in the canary
        # consumes a provider key — so a drained key must not block a CI-only PR
        # (#1159). Same trade the daily makes for the same reason (#980).
        continue-on-error: ${{ needs.detect-specs.outputs.canary == 'true' }}
        env:
          CI: "true"
          # This run is what IMPORTS the provider credentials into Langflow, so
          # the pre-flight credential check (globalSetup, #884) must not fire
          # here — it would fail on the very keys this step is about to set
          # (chicken-and-egg). Matches daily-stable.yml's Collect models step.
          PREFLIGHT_SKIP_CREDENTIALS: "1"
          # NO RETRIES for this step (#1011, ported here by #1019). CI's default
          # is 2, so a FAILING collect-models runs three full attempts, each
          # re-importing every key and re-walking the Model Providers UI against
          # the single backend this job shares with the specs. On the daily that
          # multiplier turned a ~49 s step into 7-12 min of sustained load and
          # wedged the gunicorn worker.
          #
          # The trade is NOT the daily's, and the difference is worth stating.
          # There the step is `continue-on-error: true` (#980): its result is
          # purely diagnostic, so dropping the retries costs nothing and spares
          # the shard that still has to run. Here the step is a HARD gate — a red
          # sweep fails the job outright, the specs never run, and the health gate
          # below is skipped for want of `always()`. So the cap protects no
          # downstream step; what it buys is a red in ~1 min instead of three
          # sweeps' worth of load and runner minutes. What it COSTS is the retry
          # that could have rescued a transient sweep failure (a provider 503, a
          # UI timing flake) — a real, if narrow, loss of PR-check reliability.
          # Accepted deliberately: a sweep that fails for a substantive reason
          # (drained key, component missing from the image — #1039) fails all
          # three attempts too. Revisit if PRs start going red here on flakes a
          # retry would have absorbed.
          #
          # Note the cap alone would NOT have prevented the wedges of 2026-07-28
          # (three occurrences, all after a collect-models that PASSED on its
          # first attempt in ~1 min). One successful sweep is enough to wedge the
          # backend; the health gate below is what covers that case.
          PLAYWRIGHT_RETRIES: "0"
          PLAYWRIGHT_BASE_URL: "http://localhost:7860/"
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
          MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
        run: npx playwright test tests/collect-models.spec.ts --reporter=line

      # Health gate between the two steps that share the Langflow container —
      # ported from daily-stable.yml (#1011) by #1019. collect-models' model-toggle
      # sweep can leave the backend process-wide WEDGED: container alive, event
      # loop blocked, requests simply not answered (#922, #927). globalSetup polls
      # for 120 s and then throws, so a wedge that outlasts that window costs the
      # whole job and reports itself as a preflight timeout on /api/v1/version with
      # no mention of the cause.
      #
      # On the daily this gate was documented as "attribution, with recovery as
      # upside" because no run had ever shown the backend coming back. Run
      # 30405897916 (job 90432861222) measured it — a pr-validation run that had
      # NO gate yet, so what stood in for one was globalSetup's own 120 s poll.
      # From the service container's log:
      #
      #   23:01:09  Collect models → 1 passed (57.2s)
      #   23:01:18  globalSetup starts polling /api/v1/version
      #   23:03:18  globalSetup gives up (HEALTH_TIMEOUT_MS = 120000) → job RED
      #   23:04:39  critical  WORKER TIMEOUT (pid:36)     [gunicorn.error]
      #   23:04:40  error     Worker (pid:36) was sent SIGKILL!
      #   23:04:51  replacement worker finishes startup ("Initializing agentic
      #             global variables…"), serves /auto_login at 23:04:52
      #             ← 214 s after polling began
      #
      # Read that last line for exactly what it is: recovery inferred from the
      # replacement worker's startup log, not from an observed 200 on
      # /api/v1/version — by then nothing was polling, the job had already died.
      # It is enough to establish that the backend came back, and that this step's
      # budget would have found it where globalSetup's 120 s missed by 94 s. So
      # here the gate is a RECOVERY mechanism, not only a diagnostic one — on one
      # measurement. The same log explains why the service-level --health-cmd
      # cannot cover this: gunicorn itself only noticed ~3.5 min in, and the
      # container reads healthy that whole time.
      #
      # Shared implementation since #1045 (`.github/actions/wait-for-backend`) —
      # the deadline, the heartbeat and the curl-exit-code decoding this copy
      # pioneered now apply to the daily and manual lanes too, and weekly-stable
      # finally has a gate at all. Runs only when Collect models ran: with no sweep
      # there is no wedge to wait out, an unrelated startup failure is
      # globalSetup's to report, and an LLM-free PR must not be billed for the wait.
      - name: Wait for the backend to recover from the collect-models load
        if: needs.detect-specs.outputs.needs_models == 'true'
        uses: ./.github/actions/wait-for-backend
        with:
          # Deadline comes from the action's default (420 s) — this lane is where
          # that number was argued for and it is now every lane's.
          next_step_label: "the impacted-specs run"
          attribution: "NOT a failure of the specs this PR touches"
          # No collect_models_outcome: unlike the daily and manual lanes, the step
          # here is a HARD gate (not `continue-on-error`), so a failed sweep has
          # already reddened the job and there is no surviving run to warn about.

      # Pin this lane to ONE provider's settled model (#1169). resolveTestTargets()
      # parametrizes over one model per active provider, so every impacted agent
      # spec runs an openai AND an anthropic AND a google variant here. That is
      # daily-stable.yml's job — it is the lane that owes multi-provider coverage.
      # This lane answers "does this PR break the specs it touches", which one
      # provider settles, and the difference is not cosmetic: measured 2026-07-31,
      # this workflow ran 141 times in 3.5 days, each model-needing run paying an
      # anthropic variant on claude-sonnet-5 ($3/$15 per MTok) for assertions
      # gpt-4o-mini ($0.15/$0.60) satisfies identically — 20-25x per token, with no
      # prompt caching on the anthropic side. Both the CI secret and the local key
      # drained inside that window (Anthropic credit is account-scoped, so they
      # share one balance).
      #
      # Why a script (see its header for the full argument): MODEL_TEST_PROVIDER
      # alone does not narrow the run, it runs the provider's ENTIRE catalog (the
      # dedup branch is skipped) — so the two variables must be emitted as a pair,
      # which this does. And the model must be the one collect-models settled on:
      # a hardcoded id skips silently the day the CI project loses access, leaving
      # a green PR that tested nothing (#570/#1012).
      #
      # Declines to pin — with a ::warning:: — when the provider is not active, so
      # a drained openai key costs a costlier multi-provider run rather than zero
      # coverage. Gated on needs_models for the same reason as the two steps above:
      # with no sweep there is no settled model to read.
      - name: Pin the lane to a single provider's settled model
        if: needs.detect-specs.outputs.needs_models == 'true'
        run: node scripts/select-pr-model-target.mjs --provider openai

      # In-run token consumption recorder, extended to this lane (#1183 — measure
      # the suite's real LLM spend per lane/provider). Same mechanism as
      # daily-stable.yml (#1197): poll /api/v1/monitor/traces and price what each
      # trace spent, because deleting a flow 404s its trace and this suite deletes
      # every flow it creates, so the only place to read the data is DURING the run.
      #
      # Placed here — after Collect models / the health gate / the model pin,
      # right before the actual Playwright step — so it only ever runs where
      # specs actually run: the job itself is already gated on
      # `needs.detect-specs.outputs.has_specs == 'true'`, and this step carries
      # no separate condition of its own, same as "Run impacted specs" below (an
      # earlier hard failure in this job skips both identically).
      #
      # MEASUREMENT-ONLY on this lane: --summarize (below, after the run) still
      # renders a step-summary table, but TOKENS_SUPPRESS_HISTORY keeps it from
      # writing reports/token-history.jsonl — that file is the daily's series,
      # one line per FULL @stable sweep, and this lane's scope is whatever
      # subset of specs the import graph selected for THIS PR (capped at
      # IMPACTED_SPEC_CAP, frequently zero LLM specs at all). Mixing the two
      # would corrupt the trend and the anomaly baseline the daily itself relies
      # on (see the summarizer's own comment on the knob).
      #
      # DIAGNOSTIC ONLY: continue-on-error, and it never gates the run.
      - name: Start the token consumption recorder
        shell: bash
        continue-on-error: true
        run: |
          nohup node scripts/watch-tokens.mjs > /tmp/tokens.log 2>&1 &
          echo "$!" > /tmp/tokens.pid
          disown
          echo "Token recorder started (pid $(cat /tmp/tokens.pid))."
        env:
          TOKENS_BASE_URL: http://localhost:7860
          TOKENS_OUT: token-probes.jsonl
          TOKENS_INTERVAL_MS: "15000"
          # TOKENS_TIMEOUT_MS and TOKENS_DETAIL_CAP are deliberately NOT here: they
          # are defined once at job level, because the attribution sidecar in the
          # Playwright step reads them too and a step-level env: does not reach it.
          # Below the job's timeout-minutes: 60 so this is a real backstop, not
          # dead configuration reached only after the runner already killed the
          # job. Same reasoning as the daily's TOKENS_MAX_SECONDS.
          TOKENS_MAX_SECONDS: "3300"

      - name: Run impacted specs
        env:
          CI: "true"
          # When "Collect models" is skipped (needs_models == false) the provider
          # keys below are still in the environment but were never imported into
          # Langflow as global variables, so globalSetup's credential pre-flight
          # (#884) hard-fails in CI and the job dies before running a single
          # spec — every LLM-free spec PR was guaranteed red after #953 added the
          # skip. Nothing in an LLM-free run consumes those credentials, so the
          # check has nothing to protect there: skip it exactly when the import
          # step was skipped, and keep it enforced for LLM runs.
          #
          # A canary run is the third case (#1159): the sweep DID run, but only to
          # exercise the health gate, and it is allowed to fail. Enforcing the
          # credential pre-flight there would let a drained provider key kill a
          # CI-only PR through the back door — the canary specs make no LLM call at
          # all, so there is nothing for the check to protect.
          PREFLIGHT_SKIP_CREDENTIALS: ${{ needs.detect-specs.outputs.needs_models == 'true' && needs.detect-specs.outputs.canary != 'true' && '0' || '1' }}
          PLAYWRIGHT_BASE_URL: "http://localhost:7860/"
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
          MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
          SPECS: ${{ needs.detect-specs.outputs.specs }}
          # Turns the cleanup sidecar on for this lane (#1183), same as
          # daily-stable.yml: with this unset (local runs) the helper makes no
          # request and writes no file.
          TOKENS_ATTRIB: token-attrib.jsonl
        run: |
          echo "Running specs: $SPECS"
          # shellcheck disable=SC2086
          npx playwright test $SPECS --reporter=github

      # Destructive lane (#1010). `@destructive` tests wipe account-wide state, so
      # playwright.config.ts keeps them out of the run above and pins this lane to
      # workers=1. Scoped to the same impacted $SPECS, so it is a no-op (via
      # --pass-with-no-tests) unless a destructive spec is actually impacted.
      # --reporter=github replaces the reporter list, so no HTML is written here and
      # the main run's playwright-report/ stays intact.
      - name: Run destructive lane
        if: always()
        env:
          CI: "true"
          PW_DESTRUCTIVE: "1"
          PREFLIGHT_SKIP_CREDENTIALS: ${{ needs.detect-specs.outputs.needs_models == 'true' && '0' || '1' }}
          PLAYWRIGHT_BASE_URL: "http://localhost:7860/"
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
          MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
          SPECS: ${{ needs.detect-specs.outputs.specs }}
        run: |
          # shellcheck disable=SC2086
          npx playwright test $SPECS --grep "@destructive" --pass-with-no-tests \
            --reporter=github --output=test-results-destructive

      # Stop the recorder, price what it saw, and upload the raw JSONL — mirrors
      # daily-stable.yml's shard-level "Stop and collect" + merge-job "Summarize"
      # steps, collapsed into one job since this lane has no shards to merge.
      # always(): the recorder's data is most useful precisely when the run went
      # red, and continue-on-error on every step here means a defect in the
      # reporting itself can never touch the specs' own pass/fail.
      - name: Stop and collect token consumption
        if: always()
        continue-on-error: true
        shell: bash
        run: |
          # SIGTERM is the recorder's normal exit path: it stops after the
          # current probe.
          if [ -f /tmp/tokens.pid ]; then
            kill "$(cat /tmp/tokens.pid)" 2>/dev/null || true
          fi
          # Let an in-flight append land before the file is copied — same
          # reasoning and margin as the daily's own copy of this step.
          sleep 10
          mkdir -p tokens
          cp token-probes.jsonl tokens/ 2>/dev/null || true
          cp token-attrib.jsonl tokens/ 2>/dev/null || true
          echo "--- token recorder stdout (tail) ---"
          tail -n 5 /tmp/tokens.log 2>/dev/null || true

      - name: Summarize token consumption
        if: always()
        continue-on-error: true
        run: node scripts/watch-tokens.mjs --summarize
        env:
          TOKENS_DIR: tokens
          TOKENS_PRICES: scripts/lib/model-prices.json
          WORKFLOW: pr-validation
          LANGFLOW_IMAGE: "langflowai/langflow-nightly:latest"
          # Measurement-only lane (#1183): never add a line to
          # reports/token-history.jsonl. See "Start the token consumption
          # recorder" above and the summarizer's own comment on this knob for
          # why a capped, per-PR subset is not comparable to the daily's series.
          # Suppressed openly, not silently: the summarizer states this in both
          # its log line and the step summary itself (#1012's rule).
          TOKENS_SUPPRESS_HISTORY: "1"

      - name: Upload token consumption
        uses: actions/upload-artifact@v7
        if: always()
        continue-on-error: true
        with:
          name: tokens-pr-${{ github.run_id }}
          path: tokens/
          if-no-files-found: ignore

      - name: Upload Playwright report
        uses: actions/upload-artifact@v7
        if: always()
        with:
          name: playwright-report-pr-${{ github.run_id }}
          path: playwright-report/
          retention-days: 14

      - name: Upload test results on failure
        uses: actions/upload-artifact@v7
        if: failure()
        with:
          name: test-results-pr-${{ github.run_id }}
          path: test-results/
          retention-days: 7