[aw] Failure Investigator (6h) #226
aw-failure-investigator.lock.yml
on: schedule
Annotations
5 errors and 2 warnings
|
agent
Daily workflow AIC guardrail exceeded for Matt Pocock Skills Reviewer: 5223.84072/5000.\",\n \"impact\": \"Workflow did not complete successfully and may need intervention\"\n }\n ],\n \"recommendations\": [\n {\n \"priority\": \"high\",\n \"action\": \"Review error logs to identify root cause of failure\",\n \"reason\": \"Understanding failure causes helps prevent recurrence\",\n \"example\": \"Check the errors field for specific error messages, or inspect the log files in logs_path\"\n }\n ],\n \"observability_insights\": [\n {\n \"category\": \"execution\",\n \"severity\": \"info\",\n \"title\": \"Directed execution path\",\n \"summary\": \"The run remained relatively linear with 0 turns and 0 tool types.\",\n \"evidence\": \"turns=0 tool_types=0\"\n },\n {\n \"category\": \"actuation\",\n \"severity\": \"info\",\n \"title\": \"Read-only posture observed\",\n \"summary\": \"The workflow stayed in an analysis posture and did not emit any GitHub write actions.\",\n \"evidence\": \"created_items=0 safe_items=0\"\n },\n {\n \"category\": \"execution\",\n \"severity\": \"info\",\n \"title\": \"Log template patterns mined\",\n \"summary\": \"Analysis identified 94 distinct event templates across 6 pipeline stages from 1 events.\",\n \"evidence\": \"tool_result=90 error=2 finish=1 plan=1\"\n },\n {\n \"category\": \"execution\",\n \"severity\": \"info\",\n \"title\": \"Agent stage sequence\",\n \"summary\": \"The observed pipeline stage sequence for this run.\",\n \"evidence\": \"finish\"\n }\n ],\n \"performance_metrics\": {},\n \"session_analysis\": {\n \"wall_time\": \"2.1m\",\n \"timeout_detected\": false\n },\n \"jobs\": [\n {\n \"name\": \"pre_activation\",\n \"status\": \"completed\",\n \"conclusion\": \"success\",\n \"duration\": \"15.0s\"\n },\n {\n \"name\": \"activation\",\n \"status\": \"completed\",\n \"conclusion\": \"failure\",\n \"duration\": \"52.0s\"\n },\n {\n \"name\": \"agent\",\n \"status\": \"completed\",\n \"conclusion\": \"skipped\"\n },\n {\n \"name\": \"detection\",\n \"status\": \"completed\",\n \"conclusion\": \"skipped\"\n },\n {\n \"name\": \"conclusion\",\n \"status\": \"completed\",\n \"conclusion\": \"success\",\n \"duration\": \"23.0s\"\n },\n {\n \"name\": \"safe_outputs\",\n \"status\": \"completed\",\n \"conclusion\": \"skipped\"\n }\n ],\n \"downloaded_files\": [\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585195/run_summary.json\",\n \"size\": 5736,\n \"description\": \"Cached summary of workflow run analysis\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585195/usage/agent/token_usage.jsonl\",\n \"size\": 0,\n \"description\": \"JSON Lines data file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585195/usage/detection/token_usage.jsonl\",\n \"size\": 0,\n \"description\": \"JSON Lines data file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585195/workflow-logs/0_conclusion.txt\",\n \"size\": 56978,\n \"description\": \"Text file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585195/workflow-logs/4_activation.txt\",\n \"size\": 84161,\n \"description\": \"Text file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585195/workflow-logs/5_pre_activation.txt\",\n \"size\": 29222,\n \"description\": \"Text file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585195/workflow-logs/activation/1_Set up job.txt\",\n \"size\": 1939,\n \"description\": \"Text file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585195/workflow-logs/activation/2_Checkout actions folder.txt\",\n \"size\": 18546,\n \"description\": \"Text file\"\n },\n {\n \"path
|
|
agent
Daily workflow AIC guardrail exceeded for Test Quality Sentinel: 5091.15855/5000.\",\n \"impact\": \"Workflow did not complete successfully and may need intervention\"\n }\n ],\n \"recommendations\": [\n {\n \"priority\": \"high\",\n \"action\": \"Review error logs to identify root cause of failure\",\n \"reason\": \"Understanding failure causes helps prevent recurrence\",\n \"example\": \"Check the errors field for specific error messages, or inspect the log files in logs_path\"\n }\n ],\n \"observability_insights\": [\n {\n \"category\": \"execution\",\n \"severity\": \"info\",\n \"title\": \"Directed execution path\",\n \"summary\": \"The run remained relatively linear with 0 turns and 0 tool types.\",\n \"evidence\": \"turns=0 tool_types=0\"\n },\n {\n \"category\": \"actuation\",\n \"severity\": \"info\",\n \"title\": \"Read-only posture observed\",\n \"summary\": \"The workflow stayed in an analysis posture and did not emit any GitHub write actions.\",\n \"evidence\": \"created_items=0 safe_items=0\"\n },\n {\n \"category\": \"execution\",\n \"severity\": \"info\",\n \"title\": \"Log template patterns mined\",\n \"summary\": \"Analysis identified 94 distinct event templates across 6 pipeline stages from 1 events.\",\n \"evidence\": \"finish=1 plan=1 tool_result=90 error=2\"\n },\n {\n \"category\": \"execution\",\n \"severity\": \"info\",\n \"title\": \"Agent stage sequence\",\n \"summary\": \"The observed pipeline stage sequence for this run.\",\n \"evidence\": \"finish\"\n }\n ],\n \"performance_metrics\": {},\n \"session_analysis\": {\n \"wall_time\": \"2.4m\",\n \"timeout_detected\": false\n },\n \"jobs\": [\n {\n \"name\": \"pre_activation\",\n \"status\": \"completed\",\n \"conclusion\": \"success\",\n \"duration\": \"14.0s\"\n },\n {\n \"name\": \"activation\",\n \"status\": \"completed\",\n \"conclusion\": \"failure\",\n \"duration\": \"1.0m\"\n },\n {\n \"name\": \"detection\",\n \"status\": \"completed\",\n \"conclusion\": \"skipped\"\n },\n {\n \"name\": \"conclusion\",\n \"status\": \"completed\",\n \"conclusion\": \"success\",\n \"duration\": \"29.0s\"\n },\n {\n \"name\": \"agent\",\n \"status\": \"completed\",\n \"conclusion\": \"skipped\"\n },\n {\n \"name\": \"safe_outputs\",\n \"status\": \"completed\",\n \"conclusion\": \"skipped\"\n }\n ],\n \"downloaded_files\": [\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585178/run_summary.json\",\n \"size\": 5709,\n \"description\": \"Cached summary of workflow run analysis\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585178/usage/agent/token_usage.jsonl\",\n \"size\": 0,\n \"description\": \"JSON Lines data file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585178/usage/detection/token_usage.jsonl\",\n \"size\": 0,\n \"description\": \"JSON Lines data file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585178/workflow-logs/0_conclusion.txt\",\n \"size\": 56600,\n \"description\": \"Text file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585178/workflow-logs/4_activation.txt\",\n \"size\": 91179,\n \"description\": \"Text file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585178/workflow-logs/5_pre_activation.txt\",\n \"size\": 29169,\n \"description\": \"Text file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585178/workflow-logs/activation/1_Set up job.txt\",\n \"size\": 1942,\n \"description\": \"Text file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585178/workflow-logs/activation/2_Checkout actions folder.txt\",\n \"size\": 18535,\n \"description\": \"Text file\"\n },\n {\n \"path\": \"/
|
|
agent
Daily workflow AIC guardrail exceeded ... /5000`, the `agent` job is **skipped**, and the run is marked **failure** (red run + failure-issue noise) rather than a clean no-op skip.\\n\\n### Affected workflows & representative run IDs\\n\\n| Workflow | Trigger | Daily AIC at abort | Representative failed runs |\\n|---|---|---|---|\\n| Test Quality Sentinel | pull_request | 5091 / 5000 | [§27388585178](https://github.com/github/gh-aw/actions/runs/27388585178), [§27386759277](https://github.com/github/gh-aw/actions/runs/27386759277), [§27386408010](https://github.com/github/gh-aw/actions/runs/27386408010), [§27385847787](https://github.com/github/gh-aw/actions/runs/27385847787) |\\n| Matt Pocock Skills Reviewer | pull_request | 5223 / 5000 | [§27388585195](https://github.com/github/gh-aw/actions/runs/27388585195), [§27386759271](https://github.com/github/gh-aw/actions/runs/27386408061), [§27381668091](https://github.com/github/gh-aw/actions/runs/27381668091), [§27374710086](https://github.com/github/gh-aw/actions/runs/27374710086) |\\n\\nBoth workflows had **successful runs earlier the same day** (Test Quality Sentinel [§27383667182](https://github.com/github/gh-aw/actions/runs/27383667182)), confirming the failures begin only once the cumulative daily budget is crossed.\\n\\n### Probable root cause\\n\\nThe guardrail is a **hard fail at activation**, not a graceful skip. Evidence from `audit` + `audit-diff` (failed [§27388585178](https://github.com/github/gh-aw/actions/runs/27388585178) vs successful baseline [§27383667182](https://github.com/github/gh-aw/actions/runs/27383667182)):\\n\\n<details><summary>Audit / audit-diff evidence</summary>\\n\\n- Failed run: `turns=0`, `0` GitHub API calls, `0` tokens, `agent` job **skipped** — the agent never started.\\n- Baseline run: `41` turns, `32` API calls, `api.githubcopilot.com` reached normally.\\n- Delta classification: `turns_decrease` — entire behavioral collapse is the pre-agent abort, not an agent regression.\\n- Mechanism: `GH_AW_MAX_DAILY_AI_CREDITS: ${{ vars.GH_AW_DEFAULT_MAX_DAILY_AI_CREDITS || '5000' }}` enforced in the activation job of every compiled lock file.\\n\\n</details>\\n\\nThis is **distinct** from the existing AIC issues: #38624 raises a per-run `max-ai-credits` cap (old 1000-AIC `CAPIError 429`) for one aggregator, and parent #38645 adds a soft pre-cap guard for heavy **aggregators**. Neither addresses **high-frequency PR reviewers** tripping the **activation-time daily cumulative** guardrail.\\n\\n### Proposed remediation\\n\\nDegrade the daily-guardrail breach to a clean skip instead of a failure: when `Check daily workflow token guardrail` trips, exit the run as **neutral/skipped with a `noop`-style summary** (or `continue-on-error` on that step gated to skip the agent) so it stops producing red runs and `[aw] ... failed` issues. Secondarily, allow a per-workflow override of `GH_AW_MAX_DAILY_AI_CREDITS` and raise it for PR reviewers triggered on every push.\\n\\n### Success criteria / verification\\n\\n- [ ] Daily-AIC-budget breach yields a neutral/skipped conclusion (not `failure`) and emits no failure issue.\\n- [ ] Test Quality Sentinel and Matt Pocock Skills Reviewer produce zero `failure`-conclusion runs attributable to the daily guardrail for 3 consecutive days.\\n- [ ] Per-workflow `max-ai-credits` override is documented and honored at activation.\\n\\n### Other failures observed this window (context, not yet actionable clusters)\\n\\n<details><summary>Singleton / smoke-test failures</summary>\\n\\n- **Smoke Copilot** (workflow_dispatch): `Failed to add reaction: Not Found` at activation (×3) — [§27387450863](https://github.com/github/gh-aw/actions/runs/27387450863). Likely missing comment context on manual dispatch.\\n- **Smoke Gemini** (pull_request): agent failure, 56 blocked domains incl `*.googleapis.com` — [§27387269527](https://github.com/github/gh-aw/actions/runs/27387269527). Last green Jun 6; possible firewall-allowlist drift for the Gemini engine.\\n- **PR Code Quality Reviewer**: real agent failure, 3.05M tokens
|
|
agent
Daily workflow AIC guardrail exceeded for Matt Pocock Skills Reviewer: 5223.84072/5000.\n===== 27386759268 =====\n workflow: PR Code Quality Reviewer | failed_jobs: ['agent']\n [critical] Workflow 'PR Code Quality Reviewer' failed with 1 error(s)\n [medium] Used 3.05M tokens\n [high] This Code Fix run consumed a heavy execution profile for its task shape.\n===== 27387450863 =====\n workflow: Smoke Copilot | failed_jobs: ['activation']\n [critical] Workflow 'Smoke Copilot' failed with 2 error(s): ##[error]Failed to add reaction: Not Found - https://docs.github.com/rest/reactions/reactions#create-reaction-for-an-issue-comment\n##[error]ERR_API: Failed to add reaction: Not Found - https://docs.gi...\n===== 27387269527 =====\n workflow: Smoke Gemini | failed_jobs: ['agent']\n [critical] Workflow 'Smoke Gemini' failed with 1 error(s)\n [medium] Agent attempted to access 56 blocked domains, including: *.githubusercontent.com, *.googleapis.com, *.grafana.net\n===== 27386031918 =====\n workflow: Daily Model Inventory Checker | failed_jobs: ['agent']\n [critical] Workflow 'Daily Model Inventory Checker' failed with 1 error(s)\n [medium] Used 901k tokens\n [medium] Workflow took 33 turns to complete\n [high] This General Automation run consumed a heavy execution profile for its task shape.\n [low] About 50% of this run's turns appear to be data-gathering that could move to deterministic steps.\n===== 27382565727 =====\n workflow: Daily BYOK Ollama Test | failed_jobs: ['agent']\n [critical] Workflow 'Daily BYOK Ollama Test' failed with 1 error(s)\n===== 27385890625 =====\n workflow: Agentic Commands | failed_jobs: ['route']\n [critical] Workflow 'Agentic Commands' failed with 1 error(s)\n===== 27383667211 =====\n workflow: Design Decision Gate 🏗️ | failed_jobs: ['agent', 'safe_outputs']\n [critical] Workflow 'Design Decision Gate 🏗️' failed with 2 error(s)\n [medium] Used 1.12M tokens\n [medium] Workflow took 21 turns to complete\n [high] This General Automation run consumed a heavy execution profile for its task shape.\n [low] About 50% of this run's turns appear to be data-gathering that could move to deterministic steps.","is_error":false}]},"parent_tool_use_id":null,"session_id":"c685edcf-d8e9-4699-8dd5-af5d8b67ae32","uuid":"35c6569e-f9d3-417d-8172-7db70fc9df13","timestamp":"2026-06-12T01:43:37.141Z","tool_use_result":{"stdout":"===== 27388585195 =====\n workflow: Matt Pocock Skills Reviewer | failed_jobs: ['activation']\n [critical] Workflow 'Matt Pocock Skills Reviewer' failed with 3 error(s): ##[error]Daily workflow AIC guardrail exceeded for Matt Pocock Skills Reviewer: 5223.84072/5000.\n===== 27386759268 =====\n workflow: PR Code Quality Reviewer | failed_jobs: ['agent']\n [critical] Workflow 'PR Code Quality Reviewer' failed with 1 error(s)\n [medium] Used 3.05M tokens\n [high] This Code Fix run consumed a heavy execution profile for its task shape.\n===== 27387450863 =====\n workflow: Smoke Copilot | failed_jobs: ['activation']\n [critical] Workflow 'Smoke Copilot' failed with 2 error(s): ##[error]Failed to add reaction: Not Found - https://docs.github.com/rest/reactions/reactions#create-reaction-for-an-issue-comment\n##[error]ERR_API: Failed to add reaction: Not Found - https://docs.gi...\n===== 27387269527 =====\n workflow: Smoke Gemini | failed_jobs: ['agent']\n [critical] Workflow 'Smoke Gemini' failed with 1 error(s)\n [medium] Agent attempted to access 56 blocked domains, including: *.githubusercontent.com, *.googleapis.com, *.grafana.net\n===== 27386031918 =====\n workflow: Daily Model Inventory Checker | failed_jobs: ['agent']\n [critical] Workflow 'Daily Model Inventory Checker' failed with 1 error(s)\n [medium] Used 901k tokens\n [medium] Workflow took 33 turns to complete\n [high] This General Automation run consumed a heavy execution profile for its task shape.\n [low] About 50% of this run's turns appear to be data-gathering that could move to deterministic steps.\n===== 27382565727 =====\n workflow: Daily BYOK Ollama Test | f
|
|
agent
Daily workflow AIC guardrail exceeded for Test Quality Sentinel: 5091.15855/5000.\",\n \"impact\": \"Workflow did not complete successfully and may need intervention\"\n }\n ],\n \"recommendations\": [\n {\n \"priority\": \"high\",\n \"action\": \"Review error logs to identify root cause of failure\",\n \"reason\": \"Understanding failure causes helps prevent recurrence\",\n \"example\": \"Check the errors field for specific error messages, or inspect the log files in logs_path\"\n }\n ],\n \"observability_insights\": [\n {\n \"category\": \"execution\",\n \"severity\": \"info\",\n \"title\": \"Directed execution path\",\n \"summary\": \"The run remained relatively linear with 0 turns and 0 tool types.\",\n \"evidence\": \"turns=0 tool_types=0\"\n },\n {\n \"category\": \"actuation\",\n \"severity\": \"info\",\n \"title\": \"Read-only posture observed\",\n \"summary\": \"The workflow stayed in an analysis posture and did not emit any GitHub write actions.\",\n \"evidence\": \"created_items=0 safe_items=0\"\n },\n {\n \"category\": \"execution\",\n \"severity\": \"info\",\n \"title\": \"Log template patterns mined\",\n \"summary\": \"Analysis identified 94 distinct event templates across 6 pipeline stages from 1 events.\",\n \"evidence\": \"finish=1 plan=1 tool_result=90 error=2\"\n },\n {\n \"category\": \"execution\",\n \"severity\": \"info\",\n \"title\": \"Agent stage sequence\",\n \"summary\": \"The observed pipeline stage sequence for this run.\",\n \"evidence\": \"finish\"\n }\n ],\n \"performance_metrics\": {},\n \"session_analysis\": {\n \"wall_time\": \"2.4m\",\n \"timeout_detected\": false\n },\n \"jobs\": [\n {\n \"name\": \"pre_activation\",\n \"status\": \"completed\",\n \"conclusion\": \"success\",\n \"duration\": \"14.0s\"\n },\n {\n \"name\": \"activation\",\n \"status\": \"completed\",\n \"conclusion\": \"failure\",\n \"duration\": \"1.0m\"\n },\n {\n \"name\": \"detection\",\n \"status\": \"completed\",\n \"conclusion\": \"skipped\"\n },\n {\n \"name\": \"conclusion\",\n \"status\": \"completed\",\n \"conclusion\": \"success\",\n \"duration\": \"29.0s\"\n },\n {\n \"name\": \"agent\",\n \"status\": \"completed\",\n \"conclusion\": \"skipped\"\n },\n {\n \"name\": \"safe_outputs\",\n \"status\": \"completed\",\n \"conclusion\": \"skipped\"\n }\n ],\n \"downloaded_files\": [\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585178/run_summary.json\",\n \"size\": 5709,\n \"description\": \"Cached summary of workflow run analysis\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585178/usage/agent/token_usage.jsonl\",\n \"size\": 0,\n \"description\": \"JSON Lines data file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585178/usage/detection/token_usage.jsonl\",\n \"size\": 0,\n \"description\": \"JSON Lines data file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585178/workflow-logs/0_conclusion.txt\",\n \"size\": 56600,\n \"description\": \"Text file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585178/workflow-logs/4_activation.txt\",\n \"size\": 91179,\n \"description\": \"Text file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585178/workflow-logs/5_pre_activation.txt\",\n \"size\": 29169,\n \"description\": \"Text file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585178/workflow-logs/activation/1_Set up job.txt\",\n \"size\": 1942,\n \"description\": \"Text file\"\n },\n {\n \"path\": \"/tmp/gh-aw/aw-mcp/logs/run-27388585178/workflow-logs/activation/2_Checkout actions folder.txt\",\n \"size\": 18535,\n \"description\": \"Text file\"\n },\n {\n \"path\": \"/
|
|
agent
- Line 1: create_issue 'labels' must be an array
|
|
agent
Validation errors found:
|
Artifacts
Produced during runtime
| Name | Size | Digest | |
|---|---|---|---|
|
activation
|
6.73 MB |
sha256:cd544bbcba65f07fd4563324196b3996f5a6bcb480aed2095800759eb67ba132
|
|
|
agent
|
698 KB |
sha256:c6a903392c5b5544d8c96faafd1bbb55fc45fc1c09374d443b8f3082759ef816
|
|
|
awfailureinvestigator-experiment
|
1.19 KB |
sha256:f3ccfc55b86e57032141e451ac1410ab4d555a40d9ae8fd1e78ea61ab37e696f
|
|
|
github~gh-aw~PLEAMN.dockerbuild
|
24.9 KB |
sha256:56c0ca7c05d8bbe8e4aaa4a80c727342e290851d6c48d4d6e6c3bde2d42d12c6
|
|
|
usage
|
2.13 KB |
sha256:8a63c6d197c43aa8444441410894a521bbe5bfffe6b0e9a5c5825dca13ba56dc
|
|