daily-stable.yml

# Daily Stable E2E — daily successor to weekly-stable.yml.
# Same machinery (@stable suite, QA Platform POST, failure issue) but runs every
# day at 05:00 BRT and writes its own history to reports/daily-history.jsonl
# (via the HISTORY_FILE override on the shared scripts/append-weekly-history.mjs).
# weekly-stable.yml is kept in the repo but disabled as a fallback.
name: Daily Stable E2E
run-name: "Daily Stable E2E — ${{ github.event_name == 'schedule' && 'scheduled' || github.actor }}"

on:
  schedule:
    - cron: "0 8 * * 1-5" # 05:00 BRT (UTC-3), Monday–Friday
  workflow_dispatch:
    inputs:
      langflow_image:
        description: "Langflow image repository (e.g. langflowai/langflow-nightly or langflowai/langflow)"
        required: false
        default: "langflowai/langflow-nightly"
      langflow_image_tag:
        description: "Image tag (e.g. latest, 1.5.1.dev36, 1.10.1rc3). Use a multi-arch tag — runners are amd64, so do NOT pick an -arm64 variant."
        required: false
        default: "latest"
      shards:
        description: "Number of parallel shards for the @stable run (default 4)."
        required: false
        default: "4"
      retries:
        description: "Override Playwright retries for this run (e.g. 0 for a fast, unamplified validation signal). Empty = config default (2 in CI)."
        required: false
        default: ""
      recover_timeout_s:
        description: "Seconds to wait for the backend to answer after Collect models before failing the shard (#1011). Empty = 420, the shared default of ./.github/actions/wait-for-backend (#1045)."
        required: false
        default: ""

permissions:
  issues: write
  contents: write
  # Pull the private ollama-e2e service image from GHCR with GITHUB_TOKEN.
  # An explicit permissions block defaults every unlisted scope to `none`,
  # so without this the container pull is denied (see #594).
  packages: read
  # Authenticate the Flakiness.io Playwright reporter via GitHub OIDC.
  id-token: write

jobs:
  prep:
    name: Prepare shard matrix
    # Runs inside the Playwright image so `--list` has the pinned runner without a
    # browser download (the GCS leg the runners cannot complete — see #346). No
    # Langflow service is needed: `--list` collects tests, it does not execute them.
    runs-on: ubuntu-latest
    container:
      image: mcr.microsoft.com/playwright:v1.58.2-noble
    outputs:
      # `matrix` is the strategy.matrix.include array: one entry per shard, each
      # carrying its explicit space-separated spec-file list (issue #936).
      matrix: ${{ steps.mk.outputs.matrix }}
      shard_total: ${{ steps.mk.outputs.shard_total }}
    steps:
      - uses: actions/checkout@v7

      - name: Install dependencies
        run: npm ci

      # Duration-balanced sharding (#936). Native `--shard=i/N` splits by test
      # COUNT and piles the heavy real-LLM specs onto one shard that then runs ~2x
      # longer against the single serialized Langflow backend (the load-timeout
      # root cause tracked in #773). Instead we enumerate the current @stable spec
      # files (authoritative — handles added/removed specs) and LPT bin-pack them
      # by their committed historical durations (reports/spec-durations.json). Cold
      # start / missing durations: the script degrades to a file-COUNT balance.
      - name: Compute duration-balanced shard matrix
        id: mk
        shell: bash
        env:
          SHARDS: ${{ inputs.shards || '4' }}
        run: |
          N="$SHARDS"
          case "$N" in ''|*[!0-9]*) N=4 ;; esac      # non-numeric → default 4
          if [ "$N" -lt 1 ]; then N=4; fi
          npx playwright test --grep "@stable" --list --reporter=json > /tmp/stable-list.json
          MATRIX="$(node scripts/partition-shards.mjs matrix /tmp/stable-list.json reports/spec-durations.json "$N")"
          # Extract just the include array for strategy.matrix.include.
          INCLUDE="$(node -e 'let d="";process.stdin.on("data",c=>d+=c).on("end",()=>process.stdout.write(JSON.stringify(JSON.parse(d).include)))' <<<"$MATRIX")"
          echo "matrix=$INCLUDE"  >> "$GITHUB_OUTPUT"
          echo "shard_total=$N"   >> "$GITHUB_OUTPUT"

  test:
    name: "Shard ${{ matrix.shard }}/${{ needs.prep.outputs.shard_total }} (${{ inputs.langflow_image || 'langflowai/langflow-nightly' }}:${{ inputs.langflow_image_tag || 'latest' }})"
    needs: prep
    runs-on: ubuntu-latest
    timeout-minutes: 90
    strategy:
      fail-fast: false
      # One job per shard, each carrying its own spec-file list from the
      # duration-balanced partition (prep.outputs.matrix). `matrix.shard` is the
      # 1-based index; `matrix.files` is the space-separated spec-file list (#936).
      matrix:
        include: ${{ fromJSON(needs.prep.outputs.matrix) }}
    outputs:
      langflow_version: ${{ steps.lfver.outputs.version }}

    # Run inside the official Playwright image: Chromium + all OS deps are
    # pre-installed and pulled from mcr.microsoft.com (reachable from the
    # runners), so we no longer download the browser from Google Cloud Storage
    # — the leg that the runners cannot complete (see issue #346). The tag MUST
    # match the pinned @playwright/test version in package.json.
    container:
      image: mcr.microsoft.com/playwright:v1.58.2-noble

    services:
      langflow:
        image: ${{ inputs.langflow_image || 'langflowai/langflow-nightly' }}:${{ inputs.langflow_image_tag || 'latest' }}
        ports:
          - 7860:7860
        env:
          LANGFLOW_AUTO_LOGIN: "true"
          LANGFLOW_SUPERUSER: langflow
          LANGFLOW_SUPERUSER_PASSWORD: langflow123
          # Cap the backend to one worker. Langflow's image default is (2*cpu)+1
          # workers, each holding full in-memory state; under the collect-models
          # load on the runner they contend until requests hang (#773). The launch
          # scripts have defaulted to 1 since #888, and #923 applied the same cap
          # to pr-validation.yml's service container — but that fix never reached
          # THIS workflow, so the daily kept running ~5 workers on a 2-core runner
          # that also hosts the Playwright container, Ollama and go-httpbin. That
          # is the one place the known hardening was missing when run 30351107916
          # (2026-07-28) wedged on every shard (#1011).
          # Necessary but NOT sufficient on its own: #927 proved the wedge is
          # process-wide, not worker contention. The retry cap and the health gate
          # below are the other two halves.
          LANGFLOW_WORKERS: "1"
          # Cap how long ONE wedge can cost (#1048). Langflow's default is 300 s
          # (`worker_timeout` in lfx runtime settings), handed straight to
          # gunicorn's `timeout`. The worker class is LangflowUvicornWorker — an
          # ASYNC worker — so that timeout is a heartbeat watchdog on the event
          # loop, NOT a per-request deadline: it fires when the loop stops ticking
          # (the #922/#927 wedge). Build DURATION cannot trip it — a component's
          # sync method runs off the loop in a thread (`asyncio.to_thread` in
          # `custom_component/component.py`, `_get_output_result`), so even a
          # blocking provider call keeps the heartbeat ticking and an 8-minute
          # live-LLM build is unaffected.
          #
          # Langflow's own docs contradict this: `deployment-multi-worker.mdx`
          # calls the value "how long a worker may handle a single request" and
          # says to RAISE it for long agent runs (its heavy-agent profile uses
          # 600). The code above does not support that reading. Do NOT restore a
          # higher value on the strength of those docs.
          #
          # Why lower it: a wedged worker does not recover on its own — in run
          # 30410211167 gunicorn's kill is what restored service, twice, each
          # time a full 300 s after the loop stopped (one block began within
          # seconds of a failing Google call at 00:17:28 and was killed at
          # 00:22:28). Waiting the default out buys nothing and every second of
          # it is served to unrelated specs as a dead backend.
          #
          # What 120 buys, precisely: the kill lands 60-120 s after the loop
          # stops, not at a fixed 120 — gunicorn hands the worker `timeout / 2`
          # and uvicorn refreshes the heartbeat only that often, so detection
          # costs up to one notify interval. The default's equivalent band is
          # 150-300 s.
          #
          # Measured, not assumed: a probe container of this image started with
          # LANGFLOW_WORKER_TIMEOUT=5 boots and serves /api/v1/version normally.
          # Langflow's heavy init (components, starter projects, DB) runs in the
          # PARENT process and gunicorn is launched afterwards with the app already
          # built (`__main__.py`, progress step 6), so this ceiling does not bound
          # startup — only a worker whose loop stops ticking once it is serving.
          # Rollback is this one value.
          LANGFLOW_WORKER_TIMEOUT: "120"
          # The nightly image ships LANGFLOW_ALLOW_CUSTOM_COMPONENTS=false (a
          # security default): custom-component creation is disabled, which
          # hides the sidebar "New Custom Component" button and makes
          # POST /api/v1/custom_component return 403. Enable it so the
          # custom-component @stable specs (full-custom-component,
          # customComponentAdd, api-custom-component-creation) exercise the
          # feature instead of failing on the disabled surface.
          LANGFLOW_ALLOW_CUSTOM_COMPONENTS: "true"
          # Keep tracing ON: the @stable observability/traces specs probe
          # /api/v1/monitor/traces, which is populated by the internal native
          # tracer. That tracer's worker never starts when tracing is
          # deactivated, so disabling it makes those specs fail deterministically
          # (see #352). External tracers (LangSmith/Langfuse/etc.) stay dormant
          # here because their API keys are absent.
          LANGFLOW_DEACTIVATE_TRACING: "false"
          # Enforce SQLite foreign keys (OFF by default in Langflow). Without
          # this, cascade/orphan bugs like #13955 (bulk trace delete vs. the
          # span->trace FK) are invisible — the raw DELETE "succeeds" leaving
          # orphaned rows, so traces-delete-cascade.spec.ts and any future
          # cascade guard would pass for the wrong reason. The dict replaces the
          # product default wholesale, so the default pragmas are repeated here.
          LANGFLOW_SQLITE_PRAGMAS: '{"synchronous": "NORMAL", "journal_mode": "WAL", "busy_timeout": 30000, "foreign_keys": "ON"}'
          # SSRF allowlist. Two consumers:
          #  - "ollama": the Ollama component's model-list fetch targets the
          #    sibling service by hostname; without it the nightly's SSRF
          #    protection 400s the private address — see ollama-provider.spec.ts (#583).
          #  - private CIDRs: the api-request-component specs point at the
          #    self-hosted go-httpbin service (below) by its container IP (the
          #    component's validators.url() rejects the single-label service
          #    name, so ECHO_BASE_URL must be a raw IP). The IP is whatever the
          #    Docker network assigns, so we pre-authorize the RFC-1918 ranges
          #    rather than a fixed address — the SSRF layer matches the resolved
          #    private IP against these CIDRs and skips its private-IP block
          #    (#462). This lets the echo-dependent tests run against a reliable
          #    in-CI endpoint instead of the flaky public postman-echo.
          LANGFLOW_SSRF_ALLOWED_HOSTS: "ollama,172.16.0.0/12,10.0.0.0/8,192.168.0.0/16"
        options: >-
          --health-cmd "curl -f http://localhost:7860/health_check || exit 1"
          --health-interval 15s
          --health-timeout 10s
          --health-retries 10
          --health-start-period 90s
      # Local Ollama for ollama-provider.spec.ts (§7.6), test model BAKED
      # into the image (build-ollama-image.yml) — no per-run model pull. The
      # tests skip with a reason if the service is absent/unreachable.
      ollama:
        image: ghcr.io/${{ github.repository }}/ollama-e2e:llama3.2-1b
        credentials:
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
        ports:
          - 11434:11434
        options: >-
          --health-cmd "ollama list || exit 1"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 10
          --health-start-period 20s
      # Self-hosted echo endpoint for the API Request component specs — replaces
      # the public postman-echo.com the suite defaulted to, which hard-fails the
      # daily on external outages (#462, prior recurrences #383/#407). go-httpbin
      # is an httpbin-compatible echo (paths /get, /post, /put, /patch, /delete,
      # /status/{code}, query + Host/url echo) reached BY LANGFLOW at its
      # container IP (resolved in the step below). Public GHCR image, pinned by
      # tag; pulled anonymously (no Docker Hub rate limits). No healthcheck: the
      # scratch-based image ships only the Go binary (no shell/curl), so the
      # "Resolve go-httpbin endpoint" step below both waits for and verifies it.
      # Tag has NO `v` prefix: mccutchen/go-httpbin dropped the `v` from its GHCR
      # tags at 2.17 (the `v`-prefixed series stops at v2.16.1), so `v2.23.1`
      # returns `manifest unknown` and fails container init before any test runs.
      # The pullable tag is the unprefixed `2.23.1` (#639).
      go-httpbin:
        image: ghcr.io/mccutchen/go-httpbin:2.23.1
        ports:
          - 8080:8080

    # This job runs INSIDE the Playwright container, so both the test process
    # and Langflow reach Ollama via the job network's service hostname (no
    # localhost port mapping in here — unlike manual.yml/nightly.yml).
    env:
      OLLAMA_BASE_URL: http://ollama:11434
      OLLAMA_BASE_URL_FROM_LANGFLOW: http://ollama:11434
      OLLAMA_TEST_MODEL: llama3.2:1b

    steps:
      - uses: actions/checkout@v7

      # No actions/setup-node: the Playwright image already ships the Node
      # toolchain it was built against, so we use it directly instead of
      # layering a second Node on top.
      - name: Install dependencies
        run: npm ci

      # Guard: the @playwright/test version (from npm) MUST equal the container
      # image tag, or the runner looks for a browser revision the image doesn't
      # ship and every test fails at launch with a cryptic error. Fail fast with
      # a clear message instead. Keep PLAYWRIGHT_VERSION in sync with the
      # container: image tag above.
      - name: Verify Playwright version matches the container image
        env:
          PLAYWRIGHT_VERSION: "1.58.2"
        run: |
          NPM_VERSION="$(node -p "require('@playwright/test/package.json').version")"
          if [ "$NPM_VERSION" != "$PLAYWRIGHT_VERSION" ]; then
            echo "::error::@playwright/test is $NPM_VERSION but the job runs in mcr.microsoft.com/playwright:v$PLAYWRIGHT_VERSION. Bump the container image tag and package.json together."
            exit 1
          fi
          echo "Playwright $NPM_VERSION matches the container image v$PLAYWRIGHT_VERSION."

      # No "Install Playwright browsers" step: Chromium ships in the container
      # image. npm ci installs the @playwright/test runner, whose version is
      # pinned to EXACTLY 1.58.2 in package.json to match the image tag above,
      # so the browser revision lines up. Bump both together when upgrading.

      # The async Clipboard API (and other secure-context-gated browser APIs)
      # only exist on a secure context. Chromium treats http://localhost as
      # secure but NOT the service hostname http://langflow. Inside a job
      # container the Langflow service is only reachable as http://langflow:7860,
      # so forward localhost:7860 -> langflow:7860 and keep PLAYWRIGHT_BASE_URL
      # on http://localhost:7860 — exactly as on ubuntu-latest. See issue #346.
      - name: Forward localhost:7860 to the Langflow service
        # Force bash: inside the container the default shell is `sh` (dash),
        # which lacks the `disown` builtin used below.
        shell: bash
        run: |
          apt-get update -qq && apt-get install -y -qq socat
          nohup socat TCP-LISTEN:7860,fork,reuseaddr TCP:langflow:7860 >/tmp/socat.log 2>&1 &
          disown
          for i in $(seq 1 15); do
            if curl -sf http://localhost:7860/health_check >/dev/null 2>&1; then
              echo "Forward localhost:7860 -> langflow:7860 is up."
              exit 0
            fi
            sleep 1
          done
          echo "::error::Port forward to langflow:7860 did not come up"
          cat /tmp/socat.log || true
          exit 1

      # Tag the Flakiness.io upload with the ACTUAL Langflow version under test.
      # We test against nightly:latest, whose tag never changes but whose real
      # version bumps every night (1.11.0.devN, devN+1, ...). The
      # @flakiness/playwright reporter reads FK_ENV_* env vars (prefix stripped,
      # key lowercased) as the run's "environment", so exporting
      # FK_ENV_langflow_version HERE — before the test step — makes the dashboard
      # keep a separate history per resolved version and lets us pinpoint which
      # nightly introduced a regression. Runs after the port-forward health check
      # (localhost:7860 is up) and writes to $GITHUB_ENV so the test step inherits
      # it. Fail-soft: if the version can't be read, the run still uploads, just
      # without the tag. (Mirrors the parser in the post-run "Resolve Langflow
      # version" step, which feeds the run summary.)
      - name: Tag Flakiness environment with Langflow version
        shell: bash
        run: |
          V="$(curl -sf --connect-timeout 5 --max-time 15 http://localhost:7860/api/v1/version \
            | node -e 'let d="";process.stdin.on("data",c=>d+=c).on("end",()=>{try{process.stdout.write((JSON.parse(d).version||"").toString())}catch{process.stdout.write("")}})' \
            2>/dev/null || true)"
          if [ -n "$V" ]; then
            echo "FK_ENV_langflow_version=$V" >> "$GITHUB_ENV"
            echo "Tagged Flakiness environment: langflow_version=$V"
          else
            echo "::warning::Could not resolve Langflow version; Flakiness upload will be untagged."
          fi

      # Resolve the go-httpbin service to the IP Langflow must call. The API
      # Request component runs validators.url() on the URL and REJECTS a
      # single-label host (http://go-httpbin:8080 fails), but ACCEPTS a raw IP —
      # so ECHO_BASE_URL is built from the container IP, not the service name.
      # The IP is pre-authorized by the RFC-1918 CIDRs in
      # LANGFLOW_SSRF_ALLOWED_HOSTS above, and is exported to $GITHUB_ENV so the
      # test step (and collect-models) inherit it.
      #
      # The logic moved into a shared action (#1128): it lived here only, so
      # pr-validation, nightly and manual kept calling the public host and reding
      # on its outages. `in_container: true` because this job runs INSIDE the
      # Playwright image and can resolve the service alias — the host-based lanes
      # cannot, and read the IP from `docker inspect` instead.
      #
      # `mode: warn` keeps THIS lane's behaviour exactly as it was: fail-soft, on
      # the same reasoning as `Collect models` below (#980) — a day of coverage
      # for the dozens of specs that never touch the echo outweighs strictness.
      # The PR/nightly/manual lanes use `fail`, where a silent public fallback
      # would make a third party's outage read as a product failure.
      - name: Resolve go-httpbin endpoint
        uses: ./.github/actions/resolve-echo-endpoint
        with:
          mode: warn
          in_container: "true"

      # `continue-on-error` stays: a drained provider key must not kill a day of
      # coverage for the dozens of specs that never touch that provider — the
      # lesson of #980, where a strict gate killed all four shards over a drained
      # Anthropic account. A red step here is therefore EXPECTED to be survivable,
      # and the shard deliberately does NOT abort (see the health gate below,
      # which is what actually protects the run).
      - name: Collect models
        id: collect_models
        run: npx playwright test tests/collect-models.spec.ts --reporter=line
        continue-on-error: true
        env:
          CI: "true"
          PLAYWRIGHT_BASE_URL: "http://localhost:7860/"
          # NO RETRIES for this step (#1011). CI's default is 2, so a failing
          # collect-models ran THREE full attempts, each re-importing every key and
          # re-walking the Model Providers UI against the single backend. On run
          # 30351107916 that turned a ~49 s step into 7-12 min of sustained load
          # and wedged the gunicorn worker; the next step's globalSetup preflight
          # then found nothing answering and every shard exited with 0 tests.
          # Retrying does not add information here — the first attempt already
          # produced the diagnosis, and the step is allowed to fail. The cost of a
          # transient flake is that provider specs skip this run (pre-existing
          # behaviour, and #570's gate reports it); the cost of retrying is the
          # whole run.
          PLAYWRIGHT_RETRIES: "0"
          # This run is what IMPORTS the provider credentials into Langflow, so
          # the pre-flight credential check (globalSetup, #884) must not fire
          # here — it would fail on the very keys this step is about to set.
          PREFLIGHT_SKIP_CREDENTIALS: "1"
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
          MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}

      # Health gate between the two steps that share the Langflow container
      # (#1011). collect-models' model-toggle sweep can leave the backend
      # process-wide WEDGED — container alive, event loop blocked, requests simply
      # not answered (#922, #927). globalSetup polls for 120 s and then throws, so
      # a wedge that outlasts that window costs the ENTIRE shard and reports itself
      # as a preflight error with no mention of what caused it. The gate does two
      # things the preflight cannot: it gives the wedge a longer window to clear
      # BEFORE Playwright starts, and when it does not clear it fails with the real
      # cause named, in its own step, instead of 40 specs' worth of container time
      # and a globalSetup stack trace.
      #
      # Shared implementation since #1045 — this was the first of four copies (see
      # the action for what diverged and why one of them lacked a gate entirely).
      # Two things changed for the daily when it adopted the shared version, both
      # ported from pr-validation's copy: the failure now names WHICH state it
      # observed (refused → dead vs accepted-and-unanswered → the #922/#927 wedge)
      # rather than hedging "wedged or dead", and a heartbeat keeps a step that can
      # legitimately sit for 5-7 min from reading as hung.
      #
      # Recovery is real and observed, not merely hoped for: on 2026-07-28 (#1019)
      # the service container's log showed gunicorn raising WORKER TIMEOUT ~3.5 min
      # after a sweep, SIGKILLing the worker, and the replacement serving requests
      # 214 s after polling began — 94 s past globalSetup's 120 s budget. That is
      # what the deadline is sized against; attribution is what the gate buys
      # unconditionally, on every run, whether or not the backend comes back.
      - name: Wait for the backend to recover from the collect-models load
        uses: ./.github/actions/wait-for-backend
        with:
          # 420 s for every lane since #1045 (the number's evidence is in the
          # action). The recover_timeout_s dispatch input still shortens it for a
          # validation run without editing this file; empty on schedule → 420.
          timeout_s: ${{ inputs.recover_timeout_s || '420' }}
          next_step_label: "the @stable run"
          attribution: "NOT a per-test failure"
          # Reported, never gated on — a drained key must not kill unrelated
          # specs (#980), but a silent provider skip must not go unexplained
          # either (#570).
          collect_models_outcome: ${{ steps.collect_models.outcome }}

      # Mid-run backend liveness (#1030). The gate above only proves the backend
      # is alive when Playwright STARTS. On the heavy shards the single worker
      # keeps wedging DURING the run: on run 30444299314 gunicorn killed it 7
      # times on shard 3 and 10 times on shard 4, spread evenly across ~30
      # minutes of specs. Those kills are visible only in the service container
      # log, and only 60-120 s AFTER the event loop actually stalled
      # (LANGFLOW_WORKER_TIMEOUT=120, #1048) — so an outage window inferred from
      # them is 60-120 s wide, and at that wedge frequency such windows cover
      # 33-73% of the shard. Any failure "correlates" with one by chance, which
      # is why the wedge kept costing a full triage cycle to attribute.
      #
      # Measuring from inside fixes the resolution: probe the same URL the specs
      # use every 2 s while it answers — a failed probe burns its whole 4 s
      # deadline, so the cadence relaxes to ~4 s exactly while down — and two
      # consecutive failures bound an outage to ~4 s instead of 120.
      #
      # DIAGNOSTIC ONLY — it never fails a step and never aborts the shard.
      # Aborting was this issue's original proposal and the data rejected it:
      # shard 4 wedged ten times and still passed 101 specs, so an abort would
      # discard far more coverage than it saves.
      - name: Start the backend liveness recorder (shard ${{ matrix.shard }})
        # Force bash for `disown` — the container's default shell is sh (dash).
        # Same reason as the port-forward step above.
        shell: bash
        run: |
          nohup node scripts/watch-backend.mjs > /tmp/liveness.log 2>&1 &
          echo "$!" > /tmp/liveness.pid
          disown
          echo "Backend liveness recorder started (pid $(cat /tmp/liveness.pid))."
        env:
          WATCH_URL: http://localhost:7860/api/v1/version
          WATCH_OUT: backend-liveness.jsonl
          WATCH_INTERVAL_MS: "2000"
          # A wedged worker ACCEPTS the connection and never answers, so the
          # probe needs its own deadline — without it the recorder would hang
          # exactly when it has something to record.
          WATCH_TIMEOUT_MS: "4000"
          # Backstop so a recorder that outlives its kill cannot idle for the
          # job's whole lifetime. Well above any observed shard duration (~40 min)
          # and BELOW the job's timeout-minutes: 90 — at 90 min it would only ever
          # be reached after the runner had already killed the job, making it dead
          # configuration rather than a backstop.
          WATCH_MAX_SECONDS: "3600"

      # Rotate the agent specs through ONE provider per run, by weekday (#1185).
      # The parametrized specs resolve one model per ACTIVE provider, so ~30 @stable
      # agent tests run an openai AND an anthropic AND a google variant every weekday —
      # multi-turn agent runs with the tool schemas re-sent every turn (Langflow sets no
      # `cache_control`, so nothing is cached on the anthropic side). `claude-sonnet-5`
      # is $3/$15 per MTok against `gpt-4o-mini` at ~$0.15/$0.60: 20-25x per token for
      # assertions that are about Langflow, not about the provider. The PR lane stopped
      # paying that on 2026-07-31 (#1169 / PR #1170); this is the same argument applied
      # to the lane #1170 deliberately left multi-provider.
      #
      # Coverage is not what is being cut. The provider-contract specs
      # (core-functionality/model-provider/*-provider.spec.ts) read NEITHER pin
      # variable, so every provider is still exercised end-to-end EVERY day — #1184
      # added a unit test that fails if one of them starts reading one. What rotates is
      # agent behaviour, which is a Langflow contract rather than a provider one.
      #
      # Rotation rather than a fixed pin, at identical cost: a fixed pin would make the
      # detection window for a provider-specific regression a standing human decision
      # (#643 anthropic streaming/`thinking`, #963 gemini "Message empty." — both real,
      # both caught here). Rotating bounds it to <=3 days automatically. The weekday
      # mapping is FIXED, not evenly distributed, so two Mondays are comparable:
      #   Mon openai · Tue anthropic · Wed google · Thu openai · Fri anthropic
      #
      # A drained key costs a DEVIATION, not the day: the script advances to the next
      # active provider in the rotation and says so with a ::warning::. It declines to
      # pin only when every provider is down — where the fallback is moot anyway — and
      # exits 2 on a providers.json it cannot read (#1035). Losing coverage is the more
      # expensive failure (#980), and it is not hypothetical: this lane recorded ZERO
      # tests on 07-28 and 07-31.
      #
      # continue-on-error: the pin is an optimisation. If it fails outright, the right
      # outcome is the costlier multi-provider run, not a lost day — the same trade
      # `Collect models` above already makes.
      - name: Rotate the lane to this weekday's provider
        continue-on-error: true
        run: node scripts/select-daily-model-target.mjs

      # In-run token consumption recorder (#1197). Langflow computes what each flow
      # run cost in tokens and discards it: deleting a flow 404s its trace, and this
      # suite deletes every flow it creates. So the only place to read it is during
      # the run. One request per tick — /api/v1/monitor/traces answers without a
      # flow_id — plus one detail fetch per new trace, capped per tick so a burst
      # cannot pile load onto the single backend (#817/#1048).
      #
      # DIAGNOSTIC ONLY: it never fails a step and never aborts the shard.
      - name: Start the token consumption recorder (shard ${{ matrix.shard }})
        shell: bash
        continue-on-error: true
        run: |
          nohup node scripts/watch-tokens.mjs > /tmp/tokens.log 2>&1 &
          echo "$!" > /tmp/tokens.pid
          disown
          echo "Token recorder started (pid $(cat /tmp/tokens.pid))."
        env:
          TOKENS_BASE_URL: http://localhost:7860
          TOKENS_OUT: token-probes-${{ matrix.shard }}.jsonl
          TOKENS_INTERVAL_MS: "15000"
          TOKENS_TIMEOUT_MS: "8000"
          # Same reasoning as WATCH_MAX_SECONDS: below the job's timeout-minutes so
          # it is a real backstop rather than dead configuration.
          TOKENS_MAX_SECONDS: "3600"
          TOKENS_DETAIL_CAP: "25"

      - name: Run @stable tests (shard ${{ matrix.shard }})
        # Sharded run. The reporter list is NOT overridden on the CLI: setting
        # PW_SHARD_FILE_LEVEL selects the sharded reporter shape in
        # playwright.config.ts — `blob` (the merge job rebuilds html/github/json
        # from the combined blobs) PLUS the Flakiness.io reporter. A CLI
        # `--reporter=blob` would replace the whole config list and drop the
        # Flakiness uploader; keeping it in config lets each shard upload its own
        # slice in the reporter's onExit() (per-run upload — no Flakiness merge
        # needed). OIDC auth for the upload comes from the workflow-level
        # `id-token: write` permission, inherited by this job.
        # Duration-balanced sharding (#936): this shard runs the explicit spec-file
        # list computed by prep (matrix.files), NOT Playwright's `--shard=i/N`
        # count-split. `--grep @stable` still scopes to the stable tests within
        # those files; `--pass-with-no-tests` tolerates an empty shard (N > files).
        run: npx playwright test --grep "@stable" --pass-with-no-tests ${{ matrix.files }}
        env:
          CI: "true"
          PW_SHARD_FILE_LEVEL: "1"
          # Name each shard's Flakiness.io upload so the dashboard can tell the
          # slices apart (per-test history still aggregates across shards by test).
          FLAKINESS_TITLE: "Shard ${{ matrix.shard }}/${{ needs.prep.outputs.shard_total }}"
          # Manual-dispatch retries override (empty on schedule → config default).
          PLAYWRIGHT_RETRIES: ${{ inputs.retries }}
          PLAYWRIGHT_BASE_URL: "http://localhost:7860/"
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
          MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
          # Turns the cleanup sidecar on for this lane only: with it unset (local
          # runs, the PR lane) the helper makes no request and writes no file.
          TOKENS_ATTRIB: token-attrib-${{ matrix.shard }}.jsonl

      # always(): the liveness data is most valuable precisely when the shard
      # went red, and a shard killed by its own timeout still leaves a probe log
      # worth summarizing. WATCH_FILES carries this shard's spec list into the
      # summary because the MERGED report has no shard column — without it the
      # merge job could blame shard 3's outage for a shard-4 failure (#1030).
      - name: Summarize backend liveness (shard ${{ matrix.shard }})
        if: always()
        shell: bash
        run: |
          # SIGTERM is the recorder's normal exit path: it stops after the
          # current probe. `|| true` so a recorder that already died (or never
          # started) cannot fail this step.
          if [ -f /tmp/liveness.pid ]; then
            kill "$(cat /tmp/liveness.pid)" 2>/dev/null || true
          fi
          # Let the in-flight probe land before reading the log — a probe can be
          # mid-append when the signal arrives.
          sleep 3
          mkdir -p liveness
          node scripts/watch-backend.mjs --summarize
          # Ship the raw probe log too: the summary answers "when and how long",
          # the log answers "what did the backend say", which is what a forensic
          # pass on a new wedge shape needs.
          cp backend-liveness.jsonl liveness/ 2>/dev/null || true
          echo "--- recorder stdout (tail) ---"
          tail -n 5 /tmp/liveness.log 2>/dev/null || true
        env:
          WATCH_OUT: backend-liveness.jsonl
          WATCH_SUMMARY: liveness/backend-liveness.json
          WATCH_LABEL: ${{ matrix.shard }}
          WATCH_FILES: ${{ matrix.files }}

      - name: Upload backend liveness (shard ${{ matrix.shard }})
        uses: actions/upload-artifact@v7
        if: always()
        with:
          name: liveness-${{ matrix.shard }}
          path: liveness/
          retention-days: 7
          # NOT `error` (unlike the blob upload): a shard that died before the
          # recorder ever wrote a line has no liveness data, and a missing
          # diagnostic must not turn into a red step. The merge job reports the
          # absence as `measured=false`, which it renders as UNKNOWN — never as
          # a healthy backend.
          if-no-files-found: warn

      # Renamed (#1197 review, minor fix) from a name that claimed this step
      # summarizes token consumption — it does not; it only stops the recorder
      # and copies its files. The actual pricing/summarizing happens once, in
      # the merge job's own step further down this workflow.
      - name: Stop and collect token consumption (shard ${{ matrix.shard }})
        if: always()
        continue-on-error: true
        shell: bash
        run: |
          if [ -f /tmp/tokens.pid ]; then
            kill "$(cat /tmp/tokens.pid)" 2>/dev/null || true
          fi
          # Let an in-flight append land before the file is copied. 3s was shorter
          # than a worst-case in-flight tick (TOKENS_DETAIL_CAP × TOKENS_TIMEOUT_MS
          # = 25 × 8s = 200s if every detail fetch times out sequentially), and the
          # LAST tick holds the newest traces — the ones most likely to matter for
          # an anomaly (#1197 review, minor fix). A full worst-case wait is too
          # costly to pay on every shard; 10s is a modest bump that catches an
          # ordinary slow tick without meaningfully lengthening the job.
          sleep 10
          mkdir -p tokens
          cp token-probes-${{ matrix.shard }}.jsonl tokens/ 2>/dev/null || true
          cp token-attrib-${{ matrix.shard }}.jsonl tokens/ 2>/dev/null || true
          echo "--- token recorder stdout (tail) ---"
          tail -n 5 /tmp/tokens.log 2>/dev/null || true

      - name: Upload token consumption (shard ${{ matrix.shard }})
        uses: actions/upload-artifact@v7
        if: always()
        continue-on-error: true
        with:
          name: tokens-${{ matrix.shard }}
          path: tokens/
          if-no-files-found: ignore

      # Resolve the ACTUAL Langflow version running in the service container.
      # The image tag is just `:latest` (or an RC/stable tag), so it never
      # carries the concrete nightly build (e.g. 1.11.0.dev25). Ask the running
      # service via its public /api/v1/version endpoint (AUTO_LOGIN is on, so it
      # needs no auth) and feed it into the payload so the QA Platform's Run
      # Summary can show the exact version tested. Best-effort: never fail the run.
      - name: Resolve Langflow version
        if: always()
        id: lfver
        continue-on-error: true
        run: |
          # --connect-timeout/--max-time so a slow/hung service can't stall the
          # job; tr strips any CR/LF so the value stays a single line and never
          # corrupts the key=value $GITHUB_OUTPUT format.
          V="$(curl -sf --connect-timeout 5 --max-time 15 http://localhost:7860/api/v1/version \
            | node -e 'let d="";process.stdin.on("data",c=>d+=c).on("end",()=>{try{process.stdout.write((JSON.parse(d).version||"").toString())}catch{process.stdout.write("")}})' \
            | tr -d '\r\n')"
          echo "Resolved Langflow version: '${V:-<none>}'"
          echo "version=$V" >> "$GITHUB_OUTPUT"

      - name: Upload blob report (shard ${{ matrix.shard }})
        uses: actions/upload-artifact@v7
        if: always()
        with:
          name: blob-${{ matrix.shard }}
          path: blob-report/
          retention-days: 7
          if-no-files-found: error

  merge:
    name: Merge shard reports & report
    needs: [prep, test]
    if: always()
    runs-on: ubuntu-latest
    timeout-minutes: 30
    permissions:
      issues: write
      contents: write
    container:
      image: mcr.microsoft.com/playwright:v1.58.2-noble
    steps:
      - uses: actions/checkout@v7
      - name: Install dependencies
        run: npm ci

      - name: Download all shard blobs
        uses: actions/download-artifact@v8
        with:
          pattern: blob-*
          path: all-blobs
          # Flatten every blob-<shard> artifact into all-blobs/ directly.
          # Without this, each artifact lands in its own all-blobs/blob-<shard>/
          # subdir and `merge-reports ./all-blobs` finds no report files (it does
          # not recurse). The per-shard blob zips are uniquely named
          # (report-<hash>-<shard>.zip), so flattening cannot collide.
          merge-multiple: true

      # Liveness summaries from every shard (#1030). Deliberately WITHOUT
      # merge-multiple: each shard names its file `backend-liveness.json`, so
      # flattening would have them overwrite each other. The reader recurses into
      # the per-artifact subdirectories instead.
      # continue-on-error: no shard ever uploading liveness data (an old shard
      # job, a run that died in prep) must not fail the merge — the reporter
      # renders that as UNKNOWN, which is the honest verdict.
      - name: Download shard liveness data
        uses: actions/download-artifact@v8
        if: always()
        continue-on-error: true
        with:
          pattern: liveness-*
          path: all-liveness

      # Token consumption artifacts from every shard (#1197). Flattened like the
      # blob download above: each shard names its files uniquely
      # (token-probes-<shard>.jsonl / token-attrib-<shard>.jsonl), so
      # merge-multiple cannot collide them.
      # continue-on-error: same reasoning as the liveness download — a shard that
      # never uploaded (an old job, a run that died in prep) must not fail the merge.
      - name: Download shard token consumption data
        uses: actions/download-artifact@v8
        if: always()
        continue-on-error: true
        with:
          pattern: tokens-*
          path: all-tokens
          merge-multiple: true

      - name: Guard — every expected shard produced a blob
        id: shardguard
        # always(): this guard's output gates the @stable auto-removal and the
        # incomplete-run issue, so it must be SET even when the download step
        # above failed — which is exactly what happens when every shard died
        # before producing a blob (e.g. all four failed the post-collect-models
        # health gate, #1011) and the artifact pattern matched nothing. Skipping
        # it there left `complete` empty: neither 'true' nor 'false', so the run
        # reported as a plain per-test failure with no under-count attribution.
        if: always()
        shell: bash
        run: |
          EXPECTED="${{ needs.prep.outputs.shard_total }}"
          # Tolerate a missing directory for the same reason: with zero artifacts
          # downloaded, all-blobs/ may not exist and `find` would fail the step
          # under `bash -eo pipefail`, re-opening the hole always() just closed.
          mkdir -p all-blobs
          # After merge-multiple, blobs are flat *.zip files (one per shard), not subdirs.
          FOUND="$(find all-blobs -maxdepth 1 -name '*.zip' | wc -l | tr -d ' ')"
          # An empty (or non-numeric) shard_total means `prep` ITSELF failed, so the
          # run never got a matrix and no shard could have produced anything — the
          # most incomplete a run can be. Decide that explicitly, because the
          # comparison below silently calls it COMPLETE (#1024): `[ 0 -lt "" ]`
          # errors with "integer expression expected", and a failing command inside
          # an `if` condition is not fatal under `set -e`, so it falls through to
          # the else. Defaulting EXPECTED to 0 does not help either — `0 -lt 0` is
          # false and lands in the same else. Same shape as the hole always() closed
          # in #1011: a guard asserting "complete" over a run that executed nothing.
          case "$EXPECTED" in
            ''|*[!0-9]*)
              echo "::warning::prep produced no usable shard_total ('$EXPECTED') — the run never got a shard matrix, so no shard could report ($FOUND blob(s) present). Treating the merged report as INCOMPLETE."
              echo "complete=false" >> "$GITHUB_OUTPUT"
              exit 0
              ;;
          esac
          echo "Expected $EXPECTED shard blobs, found $FOUND."
          if [ "$FOUND" -lt "$EXPECTED" ]; then
            echo "::warning::Only $FOUND/$EXPECTED shard blobs present — the merged report is INCOMPLETE (a shard died before producing a blob). Failures may be under-counted."
            echo "complete=false" >> "$GITHUB_OUTPUT"
          else
            echo "complete=true" >> "$GITHUB_OUTPUT"
          fi

      - name: Merge blob reports
        run: npx playwright merge-reports --reporter=html,github,json ./all-blobs > /dev/null
        env:
          PLAYWRIGHT_JSON_OUTPUT_NAME: results.json
          PLAYWRIGHT_HTML_REPORT: playwright-report

      # Second, independent guard (#1012). `shardguard` above answers "did every
      # shard produce a blob"; this one answers "did the run produce any TEST" —
      # a question a blob count cannot reach. On run 30351107916 (2026-07-28) all
      # four shards aborted in globalSetup on the post-collect-models backend
      # wedge (#1011) yet each still uploaded a valid, EMPTY blob: shardguard saw
      # 4/4 and reported complete, the merged report held ZERO tests, and the
      # umbrella issue rendered "No per-test @stable hard failures were
      # auto-removed" — indistinguishable from a clean triage.
      # An empty run is a DIFFERENT failure class from an incomplete merge (the
      # merge was complete; the shards just never ran anything), so it gets its
      # own output rather than being folded into `complete`. `always()` so a
      # FAILED merge-reports step is caught too — the script treats a missing
      # results.json as empty, because a guard must not go green because it could
      # not look.
      - name: Guard — merged report contains test results
        id: runguard
        if: always()
        run: node scripts/check-run-integrity.mjs
        env:
          PLAYWRIGHT_JSON: results.json

      # Name the wedge (#1030). Runs after the merge so it can attribute failing
      # attempts to the outages measured on their OWN shard, and before the
      # umbrella issue so the section can lead the body: a wedged run has to
      # announce the wedge, not a list of collateral specs that reads as
      # per-test rot.
      #
      # Reports only — it sets no gate. The @stable auto-removal below decides the
      # tag on the failure's OWN error signature (#1031), not on this table: the
      # honesty note in the section explains why overlap with an outage window is
      # a lead and not a verdict. The only thing that crosses over is the `wedged`
      # output, passed to the auto-remove action to word its exemption.
      #
      # continue-on-error: the two steps that follow and matter most on a red day
      # — `Auto-remove @stable from hard failures` and `Create issue on failure` —
      # carry NO always(), so they run under the implicit success() of every step
      # before them. A throw in this diagnostic would therefore SKIP the umbrella
      # issue it exists to improve. The script swallows its own errors too; this is
      # the second layer, for the case the failure is the `node` invocation itself.
      - name: Report mid-run backend outages
        id: liveness
        if: always()
        continue-on-error: true
        run: node scripts/report-backend-outages.mjs
        env:
          LIVENESS_DIR: all-liveness
          PLAYWRIGHT_JSON: results.json
          # Without the expected count the reporter can only speak about shards
          # that uploaded data: "2 measured shard(s)" reads the same whether the
          # run had 2 shards or 4, so a shard whose job died before writing a
          # summary would disappear instead of reading as UNKNOWN.
          SHARD_TOTAL: ${{ needs.prep.outputs.shard_total }}

      - name: Upload Playwright report (full, heavy)
        id: upload_report                     # ← full report: index.html + data/ + trace/ attachments (~380 MB)
        uses: actions/upload-artifact@v7
        if: always()
        with:
          name: playwright-report-daily-${{ github.run_id }}
          path: playwright-report/
          retention-days: 7

      # Lightweight, self-contained report: index.html embeds the whole test tree,
      # statuses, errors and steps inline (playwrightReportBase64), so it opens
      # standalone without the heavy data/ + trace/ attachments (~1.5 MB vs ~380 MB).
      # This is the artifact linked from the QA Platform (one-click, small download),
      # and it gets the longest retention GitHub allows (90 days) since it's small.
      - name: Upload report index (lightweight, long-lived)
        id: upload_index                      # ← artifact-url fed to the QA Platform payload below
        uses: actions/upload-artifact@v7
        if: always()
        with:
          name: playwright-report-index-daily-${{ github.run_id }}
          path: playwright-report/index.html
          retention-days: 90

      # Raw Playwright JSON report (the --reporter=json output produced by the
      # "Merge blob reports" step above). Uploaded UNMODIFIED — no
      # transform, no enrichment — as the machine-readable source of truth for
      # downstream per-test import/analysis: each test's status, duration,
      # retries (results[]), error, annotations and projectName plus the run's
      # stats.startTime. 90-day retention (matching the lightweight index) so a
      # late/backfill import can still reach it long after the run; the JSON is
      # small, so the long window costs ~nothing.
      - name: Upload Playwright JSON report
        uses: actions/upload-artifact@v7
        if: always()
        with:
          name: playwright-json-daily-${{ github.run_id }}
          path: results.json
          retention-days: 90

      # ── Record in the QA Platform DB: EVERY run (scheduled + manual dispatch).
      # Not gated on `schedule`, so manual runs are recorded too. Coverage is
      # best-effort; the POST is warning-only so a platform outage never fails
      # the suite / artifact / issue. ──
      - name: Compute coverage counts
        if: always()
        id: cov
        continue-on-error: true
        run: |
          echo "stable=$(npx ts-node scripts/stable-tests.ts --count)" >> "$GITHUB_OUTPUT"
          echo "total=$(grep -rE '^\s*test\s*\(' tests/tests-automations/regression --include='*.spec.ts' | wc -l | tr -d ' ')" >> "$GITHUB_OUTPUT"

      - name: Build run payload
        if: always()
        env:
          PLAYWRIGHT_JSON: results.json
          WORKFLOW: ${{ github.event_name == 'schedule' && 'daily-stable' || 'daily-stable-manual' }}
          GITHUB_RUN_ID: ${{ github.run_id }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
          LANGFLOW_IMAGE: ${{ inputs.langflow_image || 'langflowai/langflow-nightly' }}:${{ inputs.langflow_image_tag || 'latest' }}
          LANGFLOW_VERSION: ${{ needs.test.outputs.langflow_version }}
          STABLE_COUNT: ${{ steps.cov.outputs.stable }}
          TOTAL_COUNT:  ${{ steps.cov.outputs.total }}
          # Point the QA Platform at the lightweight index (one-click, small
          # download), not the heavy full report — and match its 90-day retention.
          EVIDENCE_URL: ${{ steps.upload_index.outputs.artifact-url }}
        run: |
          export EVIDENCE_EXPIRES_AT="$(date -u -d '+90 days' +%Y-%m-%dT%H:%M:%SZ)"
          node scripts/build-run-payload.mjs > payload.json
          echo "Payload:"; cat payload.json

      - name: POST run to QA Platform
        if: always()
        continue-on-error: true            # a platform failure must NOT bring down the suite / artifact / issue
        env:
          QA_PLATFORM_ENDPOINT: ${{ vars.QA_PLATFORM_ENDPOINT }}
          QA_E2E_AUTOMATION_TOKEN: ${{ secrets.QA_E2E_AUTOMATION_TOKEN }}
        run: |
          if [ -z "$QA_PLATFORM_ENDPOINT" ] || [ -z "$QA_E2E_AUTOMATION_TOKEN" ]; then
            echo "::warning::QA platform endpoint/token not configured — skipping POST."; exit 0; fi
          code=$(curl -s -o /tmp/resp.json -w '%{http_code}' -X POST "$QA_PLATFORM_ENDPOINT" \
            -H "Authorization: Bearer $QA_E2E_AUTOMATION_TOKEN" -H "Content-Type: application/json" \
            --data @payload.json)
          echo "HTTP $code"; cat /tmp/resp.json || true
          case "$code" in 200|201) echo "Recorded.";; *) echo "::warning::QA platform POST failed ($code)";; esac

      # Refresh the duration table that drives duration-balanced sharding (#936).
      # GREEN scheduled runs ONLY: a red run's per-spec times are distorted by
      # retries and timeouts, which would poison the next partition — so we only
      # trust a clean run's timings (stricter than the history file, which records
      # every run). The committed reports/spec-durations.json is read by the prep
      # job on the next daily; a missing/stale entry just falls back to a count balance.
      # ALSO gated on runguard.empty (#1012), same class of hazard as the tag
      # mutation below: `extract` on an empty report emits `{"durations":{}}` and the
      # commit step below `git add`s the file, so a zero-test run would wipe the
      # duration table on main and silently drop duration-balanced sharding on every
      # later daily. `needs.test.result == 'success'` does NOT cover that on its own —
      # the shard step runs with `--pass-with-no-tests`, so a run matching no @stable
      # test at all is GREEN with zero tests.
      # ALSO gated on runguard.partial (#1058): durations extracted from a run where
      # some shards aborted would record only the shards that ran, so the specs of a
      # dead shard silently lose their timing and the partitioner mis-balances every
      # later daily on stale data.
      - name: Refresh spec durations (green runs only)
        if: needs.test.result == 'success' && github.event_name == 'schedule' && steps.runguard.outputs.empty == 'false' && steps.runguard.outputs.partial == 'false'
        run: node scripts/partition-shards.mjs extract results.json > reports/spec-durations.json

      # Token consumption (#1197). Diagnostic: continue-on-error so a defect in the
      # cost report can never skip the steps that matter on a red day — the same
      # reasoning the liveness reporter carries.
      - name: Summarize token consumption
        if: always()
        continue-on-error: true
        run: node scripts/watch-tokens.mjs --summarize
        env:
          TOKENS_DIR: all-tokens
          TOKENS_HISTORY: reports/token-history.jsonl
          TOKENS_PRICES: scripts/lib/model-prices.json
          WORKFLOW: daily-stable
          LANGFLOW_IMAGE: ${{ inputs.langflow_image || 'langflowai/langflow-nightly' }}:${{ inputs.langflow_image_tag || 'latest' }}
          # A zero-test run is an infra abort, not a cheap day (#1012): with this at
          # 0 the summarizer writes no history line, so the abort cannot lower the
          # anomaly baseline for every run that follows. Reuses `runguard`'s own
          # count (expected+unexpected+flaky+skipped, #1012) rather than adding a
          # second computation of the same thing.
          TESTS_TOTAL: ${{ steps.runguard.outputs.tests_total }}
          # Suppress the history write on a MANUAL dispatch of this workflow
          # (#1183, found live on run 30657439522: 4 shards, one red, dispatched
          # by hand). This step still runs and still publishes its step summary
          # either way — only reports/token-history.jsonl's line is affected.
          #
          # "Append daily history" / "Commit daily history" below are already
          # gated `if: always() && github.event_name == 'schedule'`, so a manual
          # dispatch never commits that file — but this step ran unconditionally
          # and, before this line, wrote the line into the runner's ephemeral
          # workspace regardless, silently diverging from what the (uncommitted)
          # file on disk implied. A manual dispatch has an arbitrary shape (a
          # different shard count, a subset of specs, one shard failing outright)
          # that is not comparable to the daily's own fixed @stable sweep — the
          # same reasoning that keeps pr-validation.yml and manual.yml's own
          # summarize steps out of this series entirely.
          #
          # `github.event_name != 'schedule' && '1' || ''` reads as: manual
          # dispatch → "1" (suppressed), scheduled run → "" (not suppressed, the
          # knob's own truthiness check treats an empty string as "not set").
          TOKENS_SUPPRESS_HISTORY: ${{ github.event_name != 'schedule' && '1' || '' }}

      # Long-lived run history: append one JSON line per scheduled run to
      # reports/daily-history.jsonl and commit it back to main. See
      # reports/README.md for schema and queries. Runs even on failure so
      # recurring breakage is recorded, not just clean runs.
      # Gated on `schedule` only — manual dispatches (workflow_dispatch) do not
      # write to the history file, to keep the series predictable for
      # longitudinal analysis (one entry per scheduled run, same trigger, same cadence).
      - name: Append daily history
        if: always() && github.event_name == 'schedule'
        # Reuses the shared appender unchanged; the HISTORY_FILE / WORKFLOW env
        # overrides point it at the daily series, so weekly-stable.yml's script
        # and history file stay untouched.
        run: node scripts/append-weekly-history.mjs
        env:
          PLAYWRIGHT_JSON: results.json
          HISTORY_FILE: reports/daily-history.jsonl
          WORKFLOW: daily-stable
          LANGFLOW_IMAGE: ${{ inputs.langflow_image || 'langflowai/langflow-nightly' }}:${{ inputs.langflow_image_tag || 'latest' }}

      - name: Commit daily history
        if: always() && github.event_name == 'schedule'
        run: |
          # The job runs inside the Playwright container as root, while the
          # workspace is owned by the host runner uid. git 2.43 then refuses to
          # operate on the repo ("dubious ownership"). actions/checkout works
          # around this by writing safe.directory to a git global config under a
          # temporary HOME, which is gone by the time this step runs — so we
          # re-declare it here, or git reports "fatal: not in a git directory"
          # and the commit/push back to main never happens (see issue #385).
          git config --global --add safe.directory "$GITHUB_WORKSPACE"
          # `git status --porcelain` (not `git diff --quiet`) so a brand-new,
          # still-UNTRACKED reports/spec-durations.json (first green run) is
          # detected too — `git diff` ignores untracked files.
          # reports/token-history.jsonl (#1197) rides the same commit: the token
          # summarizer is continue-on-error and a zero-test run writes no line at
          # all, so this file is just as often absent/unchanged as the other two.
          if [ -z "$(git status --porcelain reports/daily-history.jsonl reports/spec-durations.json reports/token-history.jsonl 2>/dev/null)" ]; then
            echo "No history/durations change to commit."
            exit 0
          fi
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git add reports/daily-history.jsonl
          # Present only after a green run's "Refresh spec durations" step; a no-op
          # add on a red run (file unchanged / already tracked).
          [ -f reports/spec-durations.json ] && git add reports/spec-durations.json
          # Present only when the summarizer found at least one trace on a non-zero
          # test run (#1197); a no-op add otherwise.
          [ -f reports/token-history.jsonl ] && git add reports/token-history.jsonl
          git commit -m "chore(history): record daily run ${{ github.run_id }} [skip ci]"
          # Push with rebase-retry. The suite runs ~35 min, and main routinely
          # advances during that window (merged PRs, the coverage-bot [skip ci]
          # commit). A bare `git push` then dies with "! [rejected] (fetch
          # first)" and the day's history line is silently LOST — which blinds
          # the triage-dispatch automation, since its dataset builder
          # auto-discovers the run to triage from this very file (a missing line
          # makes it triage the previous run). daily-history.jsonl is
          # append-only (one line per day), so rebasing our single commit onto
          # the advanced main never conflicts. Same concurrent-push class as #741.
          for attempt in 1 2 3 4 5; do
            if git push; then
              echo "Daily history pushed (attempt $attempt)."
              exit 0
            fi
            echo "::warning::push rejected (attempt $attempt) — rebasing onto origin/main and retrying."
            git fetch origin main
            git rebase origin/main || { echo "::error::rebase of daily history onto origin/main failed"; git rebase --abort || true; exit 1; }
          done
          echo "::error::Could not push daily history after 5 attempts."
          exit 1

      # Auto-remove @stable from hard-failing tests and commit it back to main
      # (leadership decision — no human review; restoring the tag is manual). The
      # mass-failure guard inside the action leaves everything untouched when too
      # many tests fail at once (infra, not per-test rot). Scheduled runs only —
      # a manual dispatch must never mutate the test source.
      # On top of the guard, #1031: a hard failure whose last error is
      # transport-level (apiRequestContext timeout, ECONNREFUSED, the globalSetup
      # `is not reachable`) is wedge collateral, not per-test rot, and is exempted
      # INDEPENDENTLY of the threshold. The guard only ever covered the WIDE
      # wedge; a wedge costing ≤5 tests used to strip their tags with no review.
      # Gated on shardguard.complete: an incomplete merge (a dead shard) yields an
      # under-counted results.json, so never mutate the @stable tag on partial data.
      # ALSO gated on runguard.empty (#1012): a report with zero tests carries no
      # per-test evidence at all, so it must never reach the tag-mutation path. On
      # 2026-07-28 this step ran on an empty report and only found nothing to do
      # by luck — the abort happened in globalSetup, before any test could be
      # recorded as failing, so the action's mass-failure guard was never reached.
      # ALSO gated on runguard.partial (#1058). The same reasoning as `empty`, one
      # granularity down: on run 30444299314 two of four shards aborted in the
      # credentials preflight, so the report held 205 results against a ~380
      # baseline. Mutating @stable on that report would judge tags by a report that
      # never saw half the suite — and a spec cannot fail in a shard that never ran.
      - name: Auto-remove @stable from hard failures
        id: auto_remove
        if: needs.test.result == 'failure' && github.event_name == 'schedule' && steps.shardguard.outputs.complete == 'true' && steps.runguard.outputs.empty == 'false' && steps.runguard.outputs.partial == 'false'
        uses: ./.github/actions/auto-remove-stable
        with:
          playwright_json: results.json
          max_auto_remove: "5"
          run_label: "daily #${{ github.run_id }}"
          # #1030's verdict, for WORDING only. The infra-signature exemption
          # inside the action stands on the failure's own error, so it still
          # applies when this is empty — which is the run where the backend
          # state is least known and the exemption matters most.
          #
          # Gated on `measured`: a run whose shards uploaded NO liveness artifact
          # also emits `wedged=false`, and forwarding that bare would have the
          # auto-removal block claim the recorder "measured no outage" over a run
          # where it measured nothing. That is the exact misreading
          # report-backend-outages.mjs warns about ("must never be read as no
          # outage"), so unmeasured has to arrive as "" (unknown), not "false".
          backend_wedged: ${{ steps.liveness.outputs.measured == 'true' && steps.liveness.outputs.wedged || '' }}

      # Open the umbrella triage issue on failure — scheduled runs ONLY, aligned
      # with the history / auto-remove steps above. A manual dispatch is itself a
      # triage/experiment run: if it surfaces something real we open a specific
      # issue by hand following the triage rule (a hard failure, or a flaky that
      # recurs 2+ times), so an auto-umbrella on manual runs would just be noise.
      # `|| runguard.empty` (#1012): a zero-test run must open an issue even when the
      # `test` job came back GREEN. That is reachable — the shard step runs with
      # `--pass-with-no-tests`, so a run that matches no @stable test passes while
      # executing nothing. Without this the run only goes red at the last step and no
      # umbrella issue names the condition.
      - name: Create issue on failure
        if: (needs.test.result == 'failure' || steps.runguard.outputs.empty == 'true') && github.event_name == 'schedule'
        uses: actions/github-script@v9
        env:
          IMAGE: ${{ inputs.langflow_image || 'langflowai/langflow-nightly' }}:${{ inputs.langflow_image_tag || 'latest' }}
          # Set by the (schedule-only) auto-remove step above; empty only if that
          # step errored, its mass-failure guard left everything untouched, or a
          # guard skipped the step entirely (incomplete merge / empty report).
          AUTO_REMOVE_STATUS: ${{ steps.auto_remove.outputs.status }}
          AUTO_REMOVE_SUMMARY: ${{ steps.auto_remove.outputs.summary_md }}
          # From the report-integrity guard (#1012) — a run that executed ZERO
          # tests must announce itself as an infra abort, never as a per-test day.
          RUN_EMPTY: ${{ steps.runguard.outputs.empty }}
          RUN_UNREADABLE: ${{ steps.runguard.outputs.unreadable }}
          RUN_ERRORS: ${{ steps.runguard.outputs.report_errors }}
          RUN_FIRST_ERROR: ${{ steps.runguard.outputs.first_error }}
          # PARTIAL (#1058): SOME shards aborted while others ran, so the totals in
          # this body are UNDER-COUNTED. Without saying so, the umbrella reads as an
          # ordinary failure day — which is exactly how run 30444299314 reported ~184
          # unexecuted tests as a routine 10-failure triage.
          RUN_PARTIAL: ${{ steps.runguard.outputs.partial }}
          RUN_TESTS: ${{ steps.runguard.outputs.tests_total }}
          # In-run backend liveness (#1030). Rendered on EVERY umbrella issue,
          # not only on a wedged run: "the backend answered every probe" rules
          # the wedge out for the triager, and "not measured" says the state is
          # unknown. Only silence would be misleading.
          LIVENESS_MD: ${{ steps.liveness.outputs.summary_md }}
        with:
          script: |
            const today = new Date().toISOString().split('T')[0];
            const image = process.env.IMAGE;
            const arStatus = process.env.AUTO_REMOVE_STATUS || '';
            const arSummary = process.env.AUTO_REMOVE_SUMMARY || '';
            const empty = process.env.RUN_EMPTY === 'true';
            const unreadable = process.env.RUN_UNREADABLE === 'true';
            const runErrors = process.env.RUN_ERRORS || '0';
            const firstError = process.env.RUN_FIRST_ERROR || '';
            const partial = process.env.RUN_PARTIAL === 'true';
            const runTests = process.env.RUN_TESTS || '0';
            // Three shapes, most specific first.
            // 1. ZERO tests executed (#1012): there is no per-test evidence to
            //    triage, so say so instead of rendering the auto-removal line,
            //    which reads as a clean triage on an empty report.
            // 2. The auto-remove step acted — show what it did.
            // 3. Neither (it errored, or a guard skipped it) — manual triage.
            const section = empty
              ? [
                  '### ⚠️ ZERO tests executed — infra abort, not a per-test failure',
                  '',
                  unreadable
                    ? 'The merged report was **missing or unparseable** — the run produced no readable result at all. Suspect the `Merge blob reports` step and the per-shard blob artifacts first.'
                    : `The merged report carries **no test results at all** (${runErrors} top-level report error(s)) — the shards aborted before the first test.`,
                  'No spec failed and no `@stable` tag was touched, so there is **no per-test evidence to triage**.',
                  ...(firstError ? ['', '```', firstError, '```'] : []),
                  '',
                  '**Triage this as infrastructure**: find why nothing ran, not which test broke.',
                  ...(unreadable
                    ? ['Start from the `Merge blob reports` step log and the per-shard blob artifacts.']
                    : [
                        'The shard logs and the Langflow service container logs are the evidence. This does',
                        '*not* clear Langflow — a wedged or unreachable backend fails the pre-flight before',
                        'any test starts. Known cause of this shape: the post-`collect-models` backend wedge — #1011.',
                      ]),
                ]
              : partial
                ? [
                    '### ⚠️ PARTIAL run — some shards never ran their tests',
                    '',
                    `The merged report carries **${runTests} test result(s)** but also **${runErrors} top-level report error(s)**.`,
                    'A top-level error means a shard aborted before running the tests assigned to it, so',
                    'the totals above are **UNDER-COUNTED** — the specs of the dead shard are neither',
                    'passed nor failed, they simply never ran.',
                    ...(firstError ? ['', '```', firstError, '```'] : []),
                    '',
                    '`@stable` auto-removal and the spec-duration refresh were **both skipped**: a tag must',
                    'not be judged, nor a timing baseline rebuilt, on a report that never saw half the suite.',
                    '',
                    '**Triage the abort first.** Compare the recorded total against the last green run — a',
                    'large drop is the abort, not a fix. The cause above is quoted from the shard that died;',
                    'the shard logs hold the rest. Known cause of this shape: `Collect models` failing without',
                    'importing a provider key as a Langflow global variable — #1058.',
                  ]
                : arStatus
                ? ['### `@stable` auto-removal', '', arSummary]
                : [
                    '### Next steps',
                    '1. Open the Playwright report in the artifact from the run above',
                    '2. Determine if the failure is a test bug or a Langflow regression',
                    '3. If the test is incorrect or outdated: remove the `@stable` tag from the test and open a fix PR',
                    '4. If it is a Langflow regression: flag it to the team and monitor upstream',
                  ];
            // The liveness section leads the body when the backend went down:
            // the cause has to be the first thing read, ahead of the per-test
            // material, or triage starts from the collateral specs again
            // (#1030). Empty when the reporting step produced no output at all.
            const liveness = (process.env.LIVENESS_MD || '').trim();
            const livenessSection = liveness ? [liveness, ''] : [];
            // The title is what gets scanned in the issue list, so an empty run
            // must not claim that tests failed — none ran.
            const title = empty
              ? `[Daily Failure] @stable run executed ZERO tests on ${today} (${image})`
              : partial
                ? `[Daily Failure] @stable run was PARTIAL — a shard never ran on ${today} (${image})`
                : `[Daily Failure] @stable tests failed on ${today} (${image})`;
            await github.rest.issues.create({
              owner: context.repo.owner,
              repo: context.repo.repo,
              title,
              body: [
                '## Daily @stable E2E Failure',
                '',
                `- **Date:** ${today}`,
                `- **Langflow version:** \`${image}\``,
                `- **Run:** [${context.runId}](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})`,
                '',
                ...livenessSection,
                ...section,
                '',
                '/cc @Victor-w-Madeira @daniellicnerski1 @rafaelgiln',
              ].join('\n'),
              labels: ['daily-failure', 'needs-triage'],
            });

      # Runs LAST, after history / QA POST / umbrella issue have all recorded the
      # (flagged) degraded run: fail a scheduled run whose report cannot be
      # trusted, so it shows RED instead of a silent green.
      # Two distinct conditions, each with its own message (#1012):
      #   incomplete — a shard produced no blob, so results are UNDER-COUNTED;
      #   empty      — the merge was complete but ZERO tests ran (infra abort).
      # The empty case matters even though the `test` job already failed: without
      # it, nothing in the merge job ever names the condition, and the run reads
      # as an ordinary red day. Manual dispatches keep warnings only — they never
      # gate on report integrity.
      # FAIL-CLOSED on `empty != 'false'`, not `== 'true'`: an absent output means
      # runguard never reported (it was skipped, or a future edit broke its
      # entrypoint), and an unknown verdict must fail the run rather than let it
      # go green — a guard whose silence reads as "all good" is the exact failure
      # this whole mechanism exists to remove. `complete` keeps `== 'false'`: its
      # own absence is already covered, since a skipped shardguard means no blobs
      # were downloaded and runguard then sees no report at all.
      - name: Fail scheduled run on an incomplete, empty or partial report
        if: always() && github.event_name == 'schedule' && (steps.shardguard.outputs.complete == 'false' || steps.runguard.outputs.empty != 'false' || steps.runguard.outputs.partial == 'true')
        env:
          COMPLETE: ${{ steps.shardguard.outputs.complete }}
          RUN_EMPTY: ${{ steps.runguard.outputs.empty }}
          RUN_PARTIAL: ${{ steps.runguard.outputs.partial }}
          RUN_UNREADABLE: ${{ steps.runguard.outputs.unreadable }}
          RUN_ERRORS: ${{ steps.runguard.outputs.report_errors }}
          RUN_TESTS: ${{ steps.runguard.outputs.tests_total }}
          RUN_FIRST_ERROR: ${{ steps.runguard.outputs.first_error }}
        run: |
          if [ "$COMPLETE" = "false" ]; then
            echo "::error::Merge was incomplete (a shard produced no blob); results are under-counted. Failing the scheduled run so it is not mistaken for a clean pass."
          fi
          if [ "$RUN_UNREADABLE" = "true" ]; then
            echo "::error::The merged report is missing or unparseable, so ZERO test results could be read. Check the 'Merge blob reports' step and the per-shard blob artifacts."
          elif [ "$RUN_EMPTY" = "true" ]; then
            echo "::error::ZERO tests executed — the shards aborted before the first test ($RUN_ERRORS top-level report error(s), tests_total=$RUN_TESTS). This is an INFRA abort, not a per-test failure: no spec failed and no @stable tag was touched. Triage the abort, not the tests (see #1011)."
          elif [ "$RUN_PARTIAL" = "true" ]; then
            echo "::error::PARTIAL run — $RUN_TESTS test result(s) recorded, but $RUN_ERRORS top-level report error(s) mean at least one shard aborted before running its tests. The totals are UNDER-COUNTED and the specs of that shard neither passed nor failed: they never ran. @stable auto-removal and the spec-duration refresh were skipped. Cause from the shard that died: $RUN_FIRST_ERROR (see #1058)."
          elif [ -z "$RUN_EMPTY" ]; then
            echo "::error::The report-integrity guard reported nothing (empty output is unset) — its verdict is UNKNOWN, so this run cannot be trusted as a clean pass. Check the 'Guard — merged report contains test results' step."
          fi
          exit 1