test(e2e): bound downstream retries by evidence (Fixes #9166)

.github/workflows/e2e-main-retry.yaml

@@ -35,7 +35,7 @@ jobs:
     runs-on: ubuntu-latest
     timeout-minutes: 10
     permissions:
-      actions: write
+      actions: read
       contents: read
     steps:
       - name: Checkout trusted retry controller

test/e2e/README.md

@@ -19,7 +19,8 @@ before those targets run; local runners must provide it themselves.
   failures from approved `main` workflows and requests one full rerun only when
   every non-passing job has authenticated GitHub-hosted runner-loss evidence.
 - `.github/workflows/e2e-main-retry.yaml` evaluates eligible `E2E main` push
-  attempts, requests at most two failed-job reruns, and uploads attempt evidence.
+  attempts and uploads attempt evidence. It never authorizes a broad failed-job
+  rerun; retry decisions belong to bounded operation-level policies.
 - The `staging-brev-launchable` job in `.github/workflows/e2e.yaml` validates
   the baked candidate without installing or copying NemoClaw source.
 - `.github/workflows/platform-vitest-main.yaml` publishes `CI / Platform Evidence` for Ubuntu 26.04, macOS, and WSL.
@@ -667,18 +668,18 @@ It does not handle `E2E main`.
 The complete non-passing job listing must contain only authenticated hosted-runner-loss evidence for the workflow's approved runner labels.
 An ordinary assertion failure, mixed failure set, incomplete listing, custom or self-hosted label, changed evidence, or ambiguous pagination prevents recovery.
 
-For eligible `E2E main` push runs, `E2E / Main Retry` asks GitHub Actions to rerun failed jobs and their dependent jobs.
-A successful CLI artifact producer is not rerun.
-The workflow retains its CLI artifact for 3 days.
-During that period, consumers can reuse the immutable, content-addressed artifact from an earlier producer attempt in the same workflow run.
-If the artifact is unavailable when a consumer downloads it, restoration fails because the failed-job rerun does not rerun the successful producer.
-Restore validation binds the producer provenance to the workflow run, workflow SHA, and candidate checkout.
-It downloads by immutable artifact ID and verifies the manifest and the payload digest.
-It rejects a producer attempt newer than the consumer attempt.
-The controller can request two reruns.
-It does not verify that GitHub schedules a different runner, so do not treat a rerun as evidence of a fresh host.
-It ignores manual runs and source runs superseded by a newer `main` push.
-The controller checks out only trusted default-branch code and receives no repository secrets.
+For eligible `E2E main` push runs, `E2E / Main Retry` records first-attempt,
+manual-retry, and exhausted-attempt outcomes without requesting a workflow
+rerun. A failed job can represent a deterministic product assertion,
+authentication or authorization failure, policy denial, malformed input,
+ambiguous mutation, cleanup failure, or an external transient. GitHub job
+conclusions do not distinguish those classes, so a broad failed-job rerun is
+not authorized evidence. External operations use the checked-in retry inventory
+and an explicit bounded policy; new shared paths use the bounded operation
+helper. Their artifacts retain each attempt.
+Hosted runner loss remains owned by Hosted Runner Recovery. The observer ignores
+manual source runs and source runs superseded by a newer `main` push, checks out
+only trusted default-branch code, and receives no repository secrets.
 
 The runner-allocation and internal-error failures handled by Hosted Runner
 Recovery originate in GitHub Actions, outside repository-controlled workflow

test/e2e/RETRY_INVENTORY.md

@@ -0,0 +1,53 @@
+# E2E retry inventory
+
+This inventory governs operation retries, workflow reruns, and transient skips in
+the E2E suite. Readiness and eventual-consistency polling are observations, not
+operation retries: they do not repeat a mutation and retain their final probe or
+per-attempt artifacts through the existing E2E fixture APIs.
+
+Only a `transient-external` classification may retry. Authentication,
+authorization, policy denial, malformed input, deterministic product failure,
+and cleanup failure are terminal. An ambiguous mutation must reconcile external
+state before another attempt. Exhaustion remains failed.
+
+| ID | Operation and source | Failure signature | Bound and backoff | Idempotence basis | Owner | Result classification | Retained evidence | Disposition |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| `workflow-main-observer` | Completed `E2E main` run; `tools/e2e/main-run-retry.mts`, `.github/workflows/e2e-main-retry.yaml` | Any failed job | 0 automatic reruns | Not established at workflow scope | Test owner | `failed-no-retry`, `passed-first-attempt`, or `passed-after-retry` for a manual rerun | Per-attempt jobs, conclusions, duration, source SHA, and owner | Broad rerun removed; operation-level evidence is required |
+| `hosted-runner-recovery` | Confirmed GitHub-hosted runner loss; `tools/e2e/hosted-runner-recovery.mts`, `tools/e2e/hosted-runner-loss*.mts` | Authenticated runner-allocation or internal-runner evidence only | 1 recovery request; controller-defined delay | GitHub reruns a workflow attempt | GitHub Actions | Dedicated runner-loss classifications | Source and recovery run links plus authenticated job evidence | External owner; governed by #7146, not this policy |
+| `pr-rerun-reconciliation` | PR E2E dispatch reconciliation; `tools/e2e/pr-e2e-dispatch-reconciliation.mts`, `tools/e2e/pr-e2e-retry-receipt.mts` | Trusted dispatch receipt state | Contract-defined single reconciliation | Reconciles workflow and commit identity before action | GitHub Actions | Receipt-specific terminal states | Signed workflow identity and receipt | External scope; governed by #7206 |
+| `github-publication-read` | GitHub API reads; `tools/e2e/base-image-publication.mts` | Fetch error, 408, rate limit, or 5xx | 3 attempts; Retry-After/rate-limit reset or linear delay capped at 10s | Read-only | GitHub API | Returned parsed selection on success; thrown terminal HTTP/fetch error on failure or exhaustion | Caller artifact records the returned publication selection; terminal errors identify exhausted fetch or HTTP status without response content | Eligible bounded read; existing implementation retained |
+| `inference-switch-ts` | Verified inference route update; `test/e2e/fixtures/inference-switch-retry.ts` | Timeout, reset, DNS/connectivity/connect error, request transport error, or exact 502/503/504 status | 1-10 attempts; linear 5s | Setting the same desired provider/model is idempotent | Inference provider | Shared `RetryEvidence` classifications | Every attempt classification and aggregate outcome; command artifacts remain separate and redacted | Uses `runBoundedRetry`; deterministic verification mismatches stop; no `--no-verify` exhaustion bypass |
+| `inference-switch-shell` | Verified shell inference route update; `test/e2e/lib/inference-switch-retry.sh` | Same bounded transient signature as TypeScript helper | 1-10 attempts; linear 5s | Setting the same desired provider/model is idempotent | Inference provider | Exit status remains failed on exhaustion | Existing command output and retry progress | Bounded compatibility helper; no `--no-verify` exhaustion bypass |
+| `provider-install-standard` | Provider validation during Brave, cron, device-auth, Hermes-switch, network-policy, and restricted onboarding | `isTransientProviderValidationFailure` allowlist only | 1 local or 3 CI attempts; linear 10s backoff | Repeats the same desired onboarding state; restricted paths destroy the prior sandbox before retry | Inference provider | Transient allowlist versus terminal install failure | Per-attempt command artifacts; restricted paths add a terminal skip artifact | Existing bounded paths; no deterministic install retry |
+| `provider-install-agent-turn` | Agent-turn latency install; `test/e2e/live/agent-turn-latency-helpers.ts` | Same transient provider allowlist | 1-10 attempts (default 2); linear 10s backoff | Cleanup completes before repeating the same install request | Inference provider | Transient allowlist versus terminal install failure | Per-attempt install and cleanup artifacts plus progress events | Existing bounded path; no deterministic install retry |
+| `provider-install-launchable` | Launchable smoke onboarding; `test/e2e/live/launchable-smoke.test.ts` | Same transient provider allowlist | 3 attempts; linear 30s backoff | Repeats the same desired non-interactive onboarding state | Inference provider | Transient allowlist versus terminal install failure | Per-attempt onboarding artifacts and terminal CI skip evidence | Existing bounded path; no deterministic install retry |
+| `provider-ci-skip` | Cloud-provider validation skip; `test/e2e/live/cloud-inference-provider-skip.ts` | Same transient provider allowlist | No retry; one CI-only skip | Read-only provider validation | Inference provider | Explicit transient skip versus terminal failure | Skip reason and command artifact | Intentional transient skip; release conclusion unchanged |
+| `hosted-inference-budget` | Hosted inference response-budget escalation; `test/e2e/live/hermes-e2e.test.ts` | Reasoning consumed a smaller response budget | No retry; one request with the sufficient bounded budget | Read-only inference request | Test owner | Deterministic response assertion | Direct and sandbox chat artifacts | Ineligible deterministic retry removed |
+| `cloud-inference-probe` | Hosted cloud inference request; `test/e2e/live/cloud-inference.test.ts` | Curl transport timeout/reset/DNS/connect error, HTTP 429, or HTTP 5xx | 1-10 attempts (default 3); fixed 5s delay | Read-only inference request | Inference provider | Explicit external transient versus terminal response/semantic assertion | Per-request artifacts and aggregate attempt evidence | Eligible bounded read; HTTP status is separated from provider content and deterministic assertions do not retry |
+| `agent-turn-probe` | OpenClaw and Hermes agent turns; `test/e2e/live/common-egress-agent.test.ts`, `test/e2e/live/common-egress-agent-helpers.ts` | Exact OpenClaw scope-upgrade signal followed by successful recovery, or non-200 transport/provider timeout, reset, DNS, connection, 408/429/5xx signature | 3 attempts; OpenClaw linear 15s, Hermes fixed 5s | Read-only prompt; scope-upgrade retry requires successful recovery reconciliation | Agent runtime or inference provider | Explicit transient/reconciled convergence versus terminal product response | Per-attempt result, stdout, stderr, recovery artifact, and progress | Existing bounded path; HTTP 200 product replies and unconditional fallback retry remain terminal |
+| `mcp-tool-discovery` | MCP status tool discovery; `test/e2e/live/mcp-bridge-tool-discovery.ts` | Transport failure before the fixture receives a request | 2 attempts; 1s | Read-only status request | OpenShell transport | Transport-only retry versus terminal discovery assertion | Per-attempt status artifact and fixture request log | Eligible existing bounded read |
+| `mcp-restart-recovery` | MCP command after Hermes restart; `test/e2e/live/mcp-bridge-reliability.ts`, `test/e2e/live/mcp-bridge-hermes-lifecycle.ts` | Exact gateway-draining or restart transport signature after committed bridge proof | 1 restart recovery or 3 draining retries; 5s | Read-only command after committed-state verification | Hermes gateway | Exact transient signature versus terminal result | Initial and retry command artifacts | Eligible existing bounded read with precondition proof |
+| `mcp-cleanup` | MCP bridge and sandbox cleanup; `test/e2e/live/mcp-bridge-cleanup.ts` | Exact concurrent-modification delete response | 2 attempts; immediate | Delete is reconciled through registry absence/presence checks | OpenShell | Cleanup always remains visible | Cleanup ledger and final registry evidence | Reconciled cleanup retry; failure is terminal |
+| `windows-sandbox-delete` | Windows process-container cleanup; `test/e2e/live/windows-mxc-openclaw-process-container-helpers.ts` | Delete accepted but exact registry entry remains | 2 delete attempts; immediate | Registry read reconciles mutation result | OpenShell | Cleanup failure remains terminal | Delete result and exact registry evidence | Reconciled mutation retry |
+| `state-backup-destroy` | Backup/restore sandbox destruction; `test/e2e/live/state-backup-restore.test.ts` | Nonzero destroy followed by state inspection | 2 attempts; 10s | State inspection reconciles sandbox existence | OpenShell | Cleanup failure remains terminal | Per-attempt destroy artifacts | Reconciled cleanup retry |
+| `tunnel-lifecycle-observation` | Tunnel CLI URL and public reachability observation; `test/e2e/live/tunnel-lifecycle-helpers.ts` | Expected missing URL or not-yet-ready HTTP response | URL: 15 probes at 1s; public reachability: 15 probes with capped caller backoff; post-stop: 10 probes at 1s | Read-only observation after one start/stop mutation | Cloudflare tunnel | Ready URL/response or exhausted observation | Per-probe command artifact and final parsed response | No start mutation retry |
+| `mcp-quick-tunnel-establishment` | Public MCP tunnel; `test/e2e/live/mcp-bridge-servers.ts` | Child exit/error, 45s readiness deadline, or missing public origin | 3 attempts; 500ms readiness probes; linear 1s inter-attempt delay | Failed child is stopped before replacement | Cloudflare tunnel | Ready public origin or exhausted setup | Bounded diagnostic, progress events, and returned origin; raw child output excluded | Eligible bounded external setup |
+| `eventual-consistency-polling` | Generic readiness observation; `test/e2e/fixtures/polling.ts` and callers | Caller-defined expected not-ready state | Helper requires a finite `attempts` or `deadlineMs`; caller supplies explicit `delayMs` | Read-only observation | Component named by caller | Passed before bound, terminal observation, aborted, or exhausted | Last attempt and deterministic per-attempt artifact name | Not an operation retry; each caller owns and inventories its concrete bound |
+| `filesystem-cleanup` | Node `fs.rmSync` retry options; `tools/e2e/openshell-gateway-auth-artifact-safety.mts` | OS-level transient remove failure | 4 total filesystem attempts; 50ms | Idempotent removal of quarantined local artifacts | Host filesystem | Cleanup failure remains terminal | Quarantine path state without artifact content | Bounded cleanup; credentials are never serialized |
+
+## Evidence contract
+
+`test/e2e/fixtures/retry-policy.ts` emits schema version 1. The aggregate
+`outcome` is exactly one of `passed-first-attempt`, `passed-after-retry`,
+`failed-no-retry`, or `exhausted`. Cleanup failures use `failed-no-retry` with a
+final attempt whose `failureClass` is `cleanup`. Each attempt also records its
+number, failure class, reconciliation result when applicable, and whether
+another attempt was scheduled. The record deliberately excludes command output,
+errors, request bodies, headers, and environment values. Callers retain their
+normal redacted artifacts separately and may write the aggregate record through
+`onEvidence`.
+
+Adding or changing a retry, rerun, transient skip, or polling family requires an
+inventory update in the same pull request. A path is not eligible merely because
+it sometimes passes later; it needs a narrow transient signature and a stated
+idempotence or reconciliation basis.

test/e2e/fixtures/inference-switch-retry.ts

@@ -2,15 +2,16 @@
 // SPDX-License-Identifier: Apache-2.0
 
 import type { ShellProbeResult } from "./shell-probe.ts";
+import { runBoundedRetry, type RetryEvidence } from "./retry-policy.ts";
 
 const TRANSIENT_INFERENCE_SET_FAILURE =
-  /timed? out|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN|ENOTFOUND|failed to connect|error sending request|failed to verify inference endpoint|502|503|504|temporar/iu;
+  /timed? out|timeout|ETIMEDOUT|ECONNRESET|EAI_AGAIN|ENOTFOUND|failed to connect|error sending request|\b50[234]\b/iu;
 
 export function inferenceSetAttemptCount(raw: string | undefined, fallback = 3): number {
   if (raw === undefined) return fallback;
   const parsed = Number(raw);
-  if (!Number.isInteger(parsed) || parsed < 1) {
-    throw new Error(`NEMOCLAW_SWITCH_SET_ATTEMPTS must be a positive integer; got ${raw}`);
+  if (!Number.isInteger(parsed) || parsed < 1 || parsed > 10) {
+    throw new Error(`NEMOCLAW_SWITCH_SET_ATTEMPTS must be between 1 and 10; got ${raw}`);
   }
   return parsed;
 }
@@ -28,18 +29,25 @@ export async function runInferenceSetWithRetry(options: {
   attempts: number;
   delay?: (milliseconds: number) => Promise<void>;
   run: (attempt: number, verify: boolean) => Promise<ShellProbeResult>;
+  onEvidence?: (evidence: RetryEvidence) => Promise<void> | void;
 }): Promise<ShellProbeResult> {
-  const delay =
-    options.delay ??
-    ((milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)));
-  for (let attempt = 1; attempt <= options.attempts; attempt += 1) {
-    const result = await options.run(attempt, true);
-    if (result.exitCode === 0 || !isTransientInferenceSetFailure(result)) return result;
-    if (attempt < options.attempts) {
-      await delay(attempt * 5_000);
-      continue;
-    }
-    return options.run(attempt, false);
-  }
-  throw new Error("Inference switch retry loop completed without running an attempt.");
+  const execution = await runBoundedRetry({
+    operation: "inference.switch.verify",
+    owner: "inference-provider",
+    idempotence: "idempotent",
+    maxAttempts: options.attempts,
+    run: (attempt) => options.run(attempt, true),
+    classify: (result) => {
+      if (result?.exitCode === 0) return { outcome: "passed" };
+      return {
+        outcome: "failed",
+        failureClass:
+          result && isTransientInferenceSetFailure(result) ? "transient-external" : "deterministic",
+      };
+    },
+    delayMs: (attempt) => attempt * 5_000,
+    sleep: options.delay,
+    onEvidence: options.onEvidence,
+  });
+  return execution.value!;
 }