|
| 1 | +# Daily Stable E2E — daily successor to weekly-stable.yml. |
| 2 | +# Same machinery (@stable suite, QA Platform POST, failure issue) but runs every |
| 3 | +# day at 05:00 BRT and writes its own history to reports/daily-history.jsonl |
| 4 | +# (via the HISTORY_FILE override on the shared scripts/append-weekly-history.mjs). |
| 5 | +# weekly-stable.yml is kept in the repo but disabled as a fallback. |
| 6 | +name: Daily Stable E2E |
| 7 | +run-name: "Daily Stable E2E — ${{ github.event_name == 'schedule' && 'scheduled' || github.actor }}" |
| 8 | + |
| 9 | +on: |
| 10 | + schedule: |
| 11 | + - cron: "0 8 * * *" # 05:00 BRT (UTC-3), every day |
| 12 | + workflow_dispatch: |
| 13 | + inputs: |
| 14 | + langflow_image: |
| 15 | + description: "Langflow image repository (e.g. langflowai/langflow-nightly or langflowai/langflow)" |
| 16 | + required: false |
| 17 | + default: "langflowai/langflow-nightly" |
| 18 | + langflow_image_tag: |
| 19 | + description: "Image tag (e.g. latest, 1.5.1.dev36, 1.10.1rc3). Use a multi-arch tag — runners are amd64, so do NOT pick an -arm64 variant." |
| 20 | + required: false |
| 21 | + default: "latest" |
| 22 | + |
| 23 | +permissions: |
| 24 | + issues: write |
| 25 | + contents: write |
| 26 | + |
| 27 | +jobs: |
| 28 | + e2e-daily-stable: |
| 29 | + name: Daily Stable E2E (${{ inputs.langflow_image || 'langflowai/langflow-nightly' }}:${{ inputs.langflow_image_tag || 'latest' }}) |
| 30 | + runs-on: ubuntu-latest |
| 31 | + timeout-minutes: 90 |
| 32 | + |
| 33 | + # Run inside the official Playwright image: Chromium + all OS deps are |
| 34 | + # pre-installed and pulled from mcr.microsoft.com (reachable from the |
| 35 | + # runners), so we no longer download the browser from Google Cloud Storage |
| 36 | + # — the leg that the runners cannot complete (see issue #346). The tag MUST |
| 37 | + # match the pinned @playwright/test version in package.json. |
| 38 | + container: |
| 39 | + image: mcr.microsoft.com/playwright:v1.58.2-noble |
| 40 | + |
| 41 | + services: |
| 42 | + langflow: |
| 43 | + image: ${{ inputs.langflow_image || 'langflowai/langflow-nightly' }}:${{ inputs.langflow_image_tag || 'latest' }} |
| 44 | + ports: |
| 45 | + - 7860:7860 |
| 46 | + env: |
| 47 | + LANGFLOW_AUTO_LOGIN: "true" |
| 48 | + LANGFLOW_SUPERUSER: langflow |
| 49 | + LANGFLOW_SUPERUSER_PASSWORD: langflow |
| 50 | + # Keep tracing ON: the @stable observability/traces specs probe |
| 51 | + # /api/v1/monitor/traces, which is populated by the internal native |
| 52 | + # tracer. That tracer's worker never starts when tracing is |
| 53 | + # deactivated, so disabling it makes those specs fail deterministically |
| 54 | + # (see #352). External tracers (LangSmith/Langfuse/etc.) stay dormant |
| 55 | + # here because their API keys are absent. |
| 56 | + LANGFLOW_DEACTIVATE_TRACING: "false" |
| 57 | + options: >- |
| 58 | + --health-cmd "curl -f http://localhost:7860/health_check || exit 1" |
| 59 | + --health-interval 15s |
| 60 | + --health-timeout 10s |
| 61 | + --health-retries 10 |
| 62 | + --health-start-period 90s |
| 63 | +
|
| 64 | + steps: |
| 65 | + - uses: actions/checkout@v4 |
| 66 | + |
| 67 | + # No actions/setup-node: the Playwright image already ships the Node |
| 68 | + # toolchain it was built against, so we use it directly instead of |
| 69 | + # layering a second Node on top. |
| 70 | + - name: Install dependencies |
| 71 | + run: npm ci |
| 72 | + |
| 73 | + # Guard: the @playwright/test version (from npm) MUST equal the container |
| 74 | + # image tag, or the runner looks for a browser revision the image doesn't |
| 75 | + # ship and every test fails at launch with a cryptic error. Fail fast with |
| 76 | + # a clear message instead. Keep PLAYWRIGHT_VERSION in sync with the |
| 77 | + # container: image tag above. |
| 78 | + - name: Verify Playwright version matches the container image |
| 79 | + env: |
| 80 | + PLAYWRIGHT_VERSION: "1.58.2" |
| 81 | + run: | |
| 82 | + NPM_VERSION="$(node -p "require('@playwright/test/package.json').version")" |
| 83 | + if [ "$NPM_VERSION" != "$PLAYWRIGHT_VERSION" ]; then |
| 84 | + echo "::error::@playwright/test is $NPM_VERSION but the job runs in mcr.microsoft.com/playwright:v$PLAYWRIGHT_VERSION. Bump the container image tag and package.json together." |
| 85 | + exit 1 |
| 86 | + fi |
| 87 | + echo "Playwright $NPM_VERSION matches the container image v$PLAYWRIGHT_VERSION." |
| 88 | +
|
| 89 | + # No "Install Playwright browsers" step: Chromium ships in the container |
| 90 | + # image. npm ci installs the @playwright/test runner, whose version is |
| 91 | + # pinned to EXACTLY 1.58.2 in package.json to match the image tag above, |
| 92 | + # so the browser revision lines up. Bump both together when upgrading. |
| 93 | + |
| 94 | + # The async Clipboard API (and other secure-context-gated browser APIs) |
| 95 | + # only exist on a secure context. Chromium treats http://localhost as |
| 96 | + # secure but NOT the service hostname http://langflow. Inside a job |
| 97 | + # container the Langflow service is only reachable as http://langflow:7860, |
| 98 | + # so forward localhost:7860 -> langflow:7860 and keep PLAYWRIGHT_BASE_URL |
| 99 | + # on http://localhost:7860 — exactly as on ubuntu-latest. See issue #346. |
| 100 | + - name: Forward localhost:7860 to the Langflow service |
| 101 | + # Force bash: inside the container the default shell is `sh` (dash), |
| 102 | + # which lacks the `disown` builtin used below. |
| 103 | + shell: bash |
| 104 | + run: | |
| 105 | + apt-get update -qq && apt-get install -y -qq socat |
| 106 | + nohup socat TCP-LISTEN:7860,fork,reuseaddr TCP:langflow:7860 >/tmp/socat.log 2>&1 & |
| 107 | + disown |
| 108 | + for i in $(seq 1 15); do |
| 109 | + if curl -sf http://localhost:7860/health_check >/dev/null 2>&1; then |
| 110 | + echo "Forward localhost:7860 -> langflow:7860 is up." |
| 111 | + exit 0 |
| 112 | + fi |
| 113 | + sleep 1 |
| 114 | + done |
| 115 | + echo "::error::Port forward to langflow:7860 did not come up" |
| 116 | + cat /tmp/socat.log || true |
| 117 | + exit 1 |
| 118 | +
|
| 119 | + - name: Collect models |
| 120 | + run: npx playwright test tests/collect-models.spec.ts --reporter=line |
| 121 | + continue-on-error: true |
| 122 | + env: |
| 123 | + CI: "true" |
| 124 | + PLAYWRIGHT_BASE_URL: "http://localhost:7860/" |
| 125 | + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} |
| 126 | + |
| 127 | + - name: Run @stable tests |
| 128 | + run: npx playwright test --grep "@stable" --pass-with-no-tests --reporter=html,github,json |
| 129 | + env: |
| 130 | + CI: "true" |
| 131 | + PLAYWRIGHT_BASE_URL: "http://localhost:7860/" |
| 132 | + PLAYWRIGHT_JSON_OUTPUT_NAME: results.json |
| 133 | + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} |
| 134 | + |
| 135 | + - name: Upload Playwright report |
| 136 | + id: upload_report # ← id to expose artifact-url to the payload step |
| 137 | + uses: actions/upload-artifact@v4 |
| 138 | + if: always() |
| 139 | + with: |
| 140 | + name: playwright-report-daily-${{ github.run_id }} |
| 141 | + path: playwright-report/ |
| 142 | + retention-days: 14 |
| 143 | + |
| 144 | + # ── Record in the QA Platform DB: EVERY run (scheduled + manual dispatch). |
| 145 | + # Not gated on `schedule`, so manual runs are recorded too. Coverage is |
| 146 | + # best-effort; the POST is warning-only so a platform outage never fails |
| 147 | + # the suite / artifact / issue. ── |
| 148 | + - name: Compute coverage counts |
| 149 | + if: always() |
| 150 | + id: cov |
| 151 | + continue-on-error: true |
| 152 | + run: | |
| 153 | + echo "stable=$(npx ts-node scripts/stable-tests.ts --count)" >> "$GITHUB_OUTPUT" |
| 154 | + echo "total=$(grep -rE '^\s*test\s*\(' tests/tests-automations/regression --include='*.spec.ts' | wc -l | tr -d ' ')" >> "$GITHUB_OUTPUT" |
| 155 | +
|
| 156 | + - name: Build run payload |
| 157 | + if: always() |
| 158 | + env: |
| 159 | + PLAYWRIGHT_JSON: results.json |
| 160 | + WORKFLOW: ${{ github.event_name == 'schedule' && 'daily-stable' || 'daily-stable-manual' }} |
| 161 | + GITHUB_RUN_ID: ${{ github.run_id }} |
| 162 | + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} |
| 163 | + LANGFLOW_IMAGE: ${{ inputs.langflow_image || 'langflowai/langflow-nightly' }}:${{ inputs.langflow_image_tag || 'latest' }} |
| 164 | + STABLE_COUNT: ${{ steps.cov.outputs.stable }} |
| 165 | + TOTAL_COUNT: ${{ steps.cov.outputs.total }} |
| 166 | + EVIDENCE_URL: ${{ steps.upload_report.outputs.artifact-url }} |
| 167 | + run: | |
| 168 | + export EVIDENCE_EXPIRES_AT="$(date -u -d '+14 days' +%Y-%m-%dT%H:%M:%SZ)" |
| 169 | + node scripts/build-run-payload.mjs > payload.json |
| 170 | + echo "Payload:"; cat payload.json |
| 171 | +
|
| 172 | + - name: POST run to QA Platform |
| 173 | + if: always() |
| 174 | + continue-on-error: true # a platform failure must NOT bring down the suite / artifact / issue |
| 175 | + env: |
| 176 | + QA_PLATFORM_ENDPOINT: ${{ vars.QA_PLATFORM_ENDPOINT }} |
| 177 | + QA_E2E_AUTOMATION_TOKEN: ${{ secrets.QA_E2E_AUTOMATION_TOKEN }} |
| 178 | + run: | |
| 179 | + if [ -z "$QA_PLATFORM_ENDPOINT" ] || [ -z "$QA_E2E_AUTOMATION_TOKEN" ]; then |
| 180 | + echo "::warning::QA platform endpoint/token not configured — skipping POST."; exit 0; fi |
| 181 | + code=$(curl -s -o /tmp/resp.json -w '%{http_code}' -X POST "$QA_PLATFORM_ENDPOINT" \ |
| 182 | + -H "Authorization: Bearer $QA_E2E_AUTOMATION_TOKEN" -H "Content-Type: application/json" \ |
| 183 | + --data @payload.json) |
| 184 | + echo "HTTP $code"; cat /tmp/resp.json || true |
| 185 | + case "$code" in 200|201) echo "Recorded.";; *) echo "::warning::QA platform POST failed ($code)";; esac |
| 186 | +
|
| 187 | + # Long-lived run history: append one JSON line per scheduled run to |
| 188 | + # reports/daily-history.jsonl and commit it back to main. See |
| 189 | + # reports/README.md for schema and queries. Runs even on failure so |
| 190 | + # recurring breakage is recorded, not just clean runs. |
| 191 | + # Gated on `schedule` only — manual dispatches (workflow_dispatch) do not |
| 192 | + # write to the history file, to keep the series predictable for |
| 193 | + # longitudinal analysis (one entry per day, same trigger, same cadence). |
| 194 | + - name: Append daily history |
| 195 | + if: always() && github.event_name == 'schedule' |
| 196 | + # Reuses the shared appender unchanged; the HISTORY_FILE / WORKFLOW env |
| 197 | + # overrides point it at the daily series, so weekly-stable.yml's script |
| 198 | + # and history file stay untouched. |
| 199 | + run: node scripts/append-weekly-history.mjs |
| 200 | + env: |
| 201 | + PLAYWRIGHT_JSON: results.json |
| 202 | + HISTORY_FILE: reports/daily-history.jsonl |
| 203 | + WORKFLOW: daily-stable |
| 204 | + LANGFLOW_IMAGE: ${{ inputs.langflow_image || 'langflowai/langflow-nightly' }}:${{ inputs.langflow_image_tag || 'latest' }} |
| 205 | + |
| 206 | + - name: Commit daily history |
| 207 | + if: always() && github.event_name == 'schedule' |
| 208 | + run: | |
| 209 | + # The job runs inside the Playwright container as root, while the |
| 210 | + # workspace is owned by the host runner uid. git 2.43 then refuses to |
| 211 | + # operate on the repo ("dubious ownership"). actions/checkout works |
| 212 | + # around this by writing safe.directory to a git global config under a |
| 213 | + # temporary HOME, which is gone by the time this step runs — so we |
| 214 | + # re-declare it here, or git reports "fatal: not in a git directory" |
| 215 | + # and the commit/push back to main never happens (see issue #385). |
| 216 | + git config --global --add safe.directory "$GITHUB_WORKSPACE" |
| 217 | + if git diff --quiet reports/daily-history.jsonl 2>/dev/null; then |
| 218 | + echo "No history change to commit." |
| 219 | + exit 0 |
| 220 | + fi |
| 221 | + git config user.name "github-actions[bot]" |
| 222 | + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" |
| 223 | + git add reports/daily-history.jsonl |
| 224 | + git commit -m "chore(history): record daily run ${{ github.run_id }} [skip ci]" |
| 225 | + git push |
| 226 | +
|
| 227 | + - name: Create issue on failure |
| 228 | + if: failure() && github.event_name == 'schedule' |
| 229 | + uses: actions/github-script@v7 |
| 230 | + env: |
| 231 | + IMAGE: ${{ inputs.langflow_image || 'langflowai/langflow-nightly' }}:${{ inputs.langflow_image_tag || 'latest' }} |
| 232 | + with: |
| 233 | + script: | |
| 234 | + const today = new Date().toISOString().split('T')[0]; |
| 235 | + const image = process.env.IMAGE; |
| 236 | + await github.rest.issues.create({ |
| 237 | + owner: context.repo.owner, |
| 238 | + repo: context.repo.repo, |
| 239 | + title: `[Daily Failure] @stable tests failed on ${today} (${image})`, |
| 240 | + body: [ |
| 241 | + '## Daily @stable E2E Failure', |
| 242 | + '', |
| 243 | + `- **Date:** ${today}`, |
| 244 | + `- **Langflow version:** \`${image}\``, |
| 245 | + `- **Run:** [${context.runId}](${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId})`, |
| 246 | + '', |
| 247 | + '### Next steps', |
| 248 | + '1. Open the Playwright report in the artifact from the run above', |
| 249 | + '2. Determine if the failure is a test bug or a Langflow regression', |
| 250 | + '3. If the test is incorrect or outdated: remove the `@stable` tag from the test and open a fix PR', |
| 251 | + '4. If it is a Langflow regression: flag it to the team and monitor upstream', |
| 252 | + '', |
| 253 | + '/cc @Victor-w-Madeira @daniellicnerski1', |
| 254 | + ].join('\n'), |
| 255 | + labels: ['daily-failure', 'needs-triage'], |
| 256 | + }); |
0 commit comments