fix(e2e): retry exact artifact downloads

.github/workflows/e2e.yaml

@@ -168,13 +168,12 @@ jobs:
 
       - name: Download immutable Deep Agents Code base contract
         if: ${{ steps.publication_mode.outputs.required == '1' }}
-        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
-        with:
-          github-token: ${{ github.token }}
-          name: managed-base-${{ steps.publication.outputs.run_id }}-${{ steps.publication.outputs.run_attempt }}-langchain-deepagents-code
-          path: ${{ runner.temp }}/dcode-base-contract
-          repository: NVIDIA/NemoClaw
-          run-id: ${{ steps.publication.outputs.run_id }}
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+          PUBLICATION_HEAD_SHA: ${{ steps.publication.outputs.head_sha }}
+          PUBLICATION_RUN_ATTEMPT: ${{ steps.publication.outputs.run_attempt }}
+          PUBLICATION_RUN_ID: ${{ steps.publication.outputs.run_id }}
+        run: node --experimental-strip-types --no-warnings tools/e2e/exact-artifact-download.mts "${RUNNER_TEMP}/dcode-base-contract"
 
       - id: validate_dcode_base
         name: Validate immutable Deep Agents Code base

test/e2e/RETRY_INVENTORY.md

@@ -18,6 +18,7 @@ Exhaustion remains failed.
 | `pr-rerun-reconciliation` | PR E2E dispatch reconciliation; `tools/e2e/pr-e2e-dispatch-reconciliation.mts`, `tools/e2e/pr-e2e-retry-receipt.mts` | Trusted dispatch receipt state | Contract-defined single reconciliation | Reconciles workflow and commit identity before action | GitHub Actions | Receipt-specific terminal states | Signed workflow identity and receipt | External scope; governed by #7206 |
 | `github-publication-read` | GitHub API reads; `tools/e2e/base-image-publication.mts` | Fetch error, 408, rate limit, or 5xx | 3 attempts; Retry-After/rate-limit reset or linear delay capped at 10s | Read-only | GitHub API | Returned parsed selection on success; thrown terminal HTTP/fetch error on failure or exhaustion | Caller artifact records the returned publication selection; terminal errors identify exhausted fetch or HTTP status without response content | Eligible bounded read; existing implementation retained |
 | `trusted-controller-collaborator-permission-read` | Collaborator-permission reads for manual PR dispatch, release waiver, and Launchable publication; `.github/workflows/e2e.yaml` | Curl exit 5, 6, 7, 16, 18, 28, 35, 52, 55, 56, 92, 95, or 96; HTTP 408, 429, or 5xx | 3 attempts; linear 1s then 2s | Read-only GitHub API request | GitHub API | Transient API read versus terminal authentication, authorization, actor, or response failure | Operation name, attempt number, and sanitized failure class or HTTP status; no response body, header, or token | Eligible bounded read; HTTP 401, 403, 404, and 422, malformed responses, actor failures, and insufficient roles remain terminal; no cached permission or workflow rerun |
+| `github-exact-artifact-content-read` | Bound Deep Agents Code contract artifact; `tools/e2e/exact-artifact-download.mts` | Transport failure, HTTP 408, HTTP 429, or HTTP 5xx while reading one pre-bound artifact ID | 3 attempts; Retry-After or linear delay capped at 10s | Read-only request against one immutable artifact ID, name, size, digest, producer run, attempt, and head | GitHub artifact service | `passed-first-attempt`, `passed-after-retry`, `exhausted` for transient exhaustion, or `failed-no-retry` for terminal HTTP; identity, size, digest, archive, and contract failures throw without an aggregate outcome or `failureClass` | Content-read attempts log only the sanitized operation, attempt, HTTP status or transport class, and outcome; thrown validation failures expose only their bounded error message, never headers, body, token, signed URL, or artifact content | Standalone bounded content read; it does not use `retry-policy.ts` or `RetryEvidence`, and all identity, integrity, archive, and contract failures remain terminal |
 | `inference-switch-ts` | Verified inference route update; `test/e2e/fixtures/inference-switch-retry.ts` | Timeout, reset, DNS/connectivity/connect error, request transport error, or exact 502/503/504 status; authentication, authorization, policy, malformed-input, and invalid-request signals take precedence | 1-10 attempts; linear 5s | Setting the same desired provider/model is idempotent | Inference provider | Shared `RetryEvidence` classifications | Every attempt classification and aggregate outcome; command artifacts remain separate and redacted | Uses `runBoundedRetry`; deterministic verification mismatches stop; no `--no-verify` exhaustion bypass |
 | `inference-switch-shell` | Verified shell inference route update; `test/e2e/lib/inference-switch-retry.sh` | Same bounded transient and terminal-precedence signatures as the TypeScript helper | 1-10 attempts; linear 5s | Setting the same desired provider/model is idempotent | Inference provider | Exit status remains failed on exhaustion | Existing command output and retry progress | Bounded compatibility helper; no `--no-verify` exhaustion bypass |
 | `provider-install-standard` | Provider validation during Brave, cron, device-auth, Hermes-switch, network-policy, and restricted onboarding | `isTransientProviderValidationFailure` allowlist only | 1 local or 3 CI attempts; linear 10s backoff | Repeats the same desired onboarding state; restricted paths destroy the prior sandbox before retry | Inference provider | Transient allowlist versus terminal install failure | Per-attempt command artifacts; restricted paths add a terminal skip artifact | Existing bounded paths; no deterministic install retry |

test/e2e/support/base-image-publication-workflow-boundary.test.ts

@@ -60,6 +60,13 @@ function gateSteps(value: MutableWorkflow): MutableStep[] {
   );
 }
 
+function gateStep(value: MutableWorkflow, name: string): MutableStep {
+  return required(
+    gateSteps(value).find((step) => step.name === name),
+    `base-image-publication test fixture is missing step ${name}`,
+  );
+}
+
 function runClassifier(environment: {
   checkoutSha: string;
   eventName: string;
@@ -181,12 +188,17 @@ describe("base-image publication workflow boundary (#7372)", () => {
       },
     ],
     [
-      "contract download pin",
-      (value) => (gateSteps(value)[4].uses = "actions/download-artifact@v8"),
+      "contract download command",
+      (value) =>
+        (gateStep(value, "Download immutable Deep Agents Code base contract").run =
+          "node unreviewed.mts"),
     ],
     [
       "contract run binding",
-      (value) => (gateSteps(value)[4].with!["run-id"] = "${{ github.run_id }}"),
+      (value) =>
+        (gateStep(value, "Download immutable Deep Agents Code base contract").env![
+          "PUBLICATION_RUN_ID"
+        ] = "${{ github.run_id }}"),
     ],
     [
       "contract validation",

test/e2e/support/e2e-collaborator-permission-retry.test.ts

@@ -36,9 +36,9 @@ const AUTHORIZATION_STEPS: AuthorizationStep[] = [
     name: "Authorize release qualification waiver",
   },
   {
-    deniedMessage: "Launchable image publication requires a repository maintainer or administrator",
-    mismatchMessage: "Launchable image publication permission response did not match the actor",
-    name: "Authorize Launchable image publication",
+    deniedMessage: "Launchable E2E requires a repository maintainer or administrator",
+    mismatchMessage: "Launchable E2E permission response did not match the actor",
+    name: "Authorize Launchable E2E maintainer dispatch",
   },
 ];
 

test/e2e/support/exact-artifact-download.test.ts

@@ -0,0 +1,282 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { createHash } from "node:crypto";
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+import { describe, expect, it, vi } from "vitest";
+
+import {
+  bindExactArtifact,
+  downloadBoundArtifact,
+  exactArtifactName,
+  materializeContractArchive,
+  type BoundArtifactIdentity,
+  type ExactArtifactExpectation,
+} from "../../../tools/e2e/exact-artifact-download.mts";
+import { validateDcodeBaseImageContract } from "../../../tools/e2e/dcode-base-image-contract.mts";
+import { artifactZip } from "../../helpers/artifact-zip";
+
+const EXPECTED: ExactArtifactExpectation = {
+  headSha: "a".repeat(40),
+  runAttempt: 2,
+  runId: 7001,
+};
+
+function archive(contents = "{}\n"): Buffer {
+  return artifactZip([{ name: "contract.json", contents }]);
+}
+
+function metadata(bytes: Buffer, overrides: Record<string, unknown> = {}): unknown {
+  const id = 9001;
+  return {
+    total_count: 1,
+    artifacts: [
+      {
+        id,
+        name: exactArtifactName(EXPECTED),
+        size_in_bytes: bytes.length,
+        expired: false,
+        digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}`,
+        archive_download_url: `https://api.github.com/repos/NVIDIA/NemoClaw/actions/artifacts/${id}/zip`,
+        workflow_run: { id: EXPECTED.runId, head_sha: EXPECTED.headSha },
+        ...overrides,
+      },
+    ],
+  };
+}
+
+function identity(bytes = archive()): BoundArtifactIdentity {
+  return bindExactArtifact(metadata(bytes), EXPECTED);
+}
+
+function response(bytes: Buffer): Response {
+  return new Response(new Uint8Array(bytes), {
+    status: 200,
+    headers: { "content-length": String(bytes.length) },
+  });
+}
+
+function parseArtifactReadEvidence(message: string): Record<string, string> {
+  const [operation, ...fields] = message.trim().split(/\s+/u);
+  return {
+    operation,
+    ...Object.fromEntries(fields.map((field) => field.split("=", 2))),
+  };
+}
+
+describe("exact artifact download (#9340)", () => {
+  it("binds immutable identity before the content read", () => {
+    const bytes = archive();
+    expect(bindExactArtifact(metadata(bytes), EXPECTED)).toEqual({
+      ...EXPECTED,
+      archivePath: "/repos/NVIDIA/NemoClaw/actions/artifacts/9001/zip",
+      digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}`,
+      id: 9001,
+      name: exactArtifactName(EXPECTED),
+      size: bytes.length,
+    });
+  });
+
+  it.each([
+    ["expired", { expired: true }, "non-expired"],
+    ["artifact id URL", { id: 9002 }, "archive URL does not match artifact id"],
+    ["name", { name: "another-artifact" }, "missing or ambiguous"],
+    ["digest", { digest: "sha256:invalid" }, "digest is invalid"],
+    [
+      "run",
+      { workflow_run: { id: 7002, head_sha: EXPECTED.headSha } },
+      "producer run does not match",
+    ],
+    [
+      "head",
+      { workflow_run: { id: EXPECTED.runId, head_sha: "b".repeat(40) } },
+      "producer head does not match",
+    ],
+    [
+      "archive URL",
+      { archive_download_url: "https://example.com/artifact.zip" },
+      "archive URL does not match artifact id",
+    ],
+  ])("rejects %s identity drift", (_field, overrides, message) => {
+    expect(() => bindExactArtifact(metadata(archive(), overrides), EXPECTED)).toThrow(message);
+  });
+
+  it("rejects ambiguous artifact metadata", () => {
+    const value = metadata(archive()) as { artifacts: unknown[]; total_count: number };
+    value.artifacts.push(value.artifacts[0]);
+    value.total_count = 2;
+    expect(() => bindExactArtifact(value, EXPECTED)).toThrow("missing or ambiguous");
+  });
+
+  it("retries one transient response against the same artifact and honors Retry-After", async () => {
+    const bytes = archive();
+    const fetchImpl = vi
+      .fn<(input: string, init: RequestInit) => Promise<Response>>()
+      .mockResolvedValueOnce(
+        new Response("sensitive upstream body", { status: 503, headers: { "retry-after": "2" } }),
+      )
+      .mockResolvedValueOnce(response(bytes));
+    const sleep = vi.fn<(milliseconds: number) => Promise<void>>().mockResolvedValue();
+    const log = vi.fn<(message: string) => void>();
+
+    await expect(
+      downloadBoundArtifact(identity(bytes), "secret-token", { fetchImpl, log, sleep }),
+    ).resolves.toEqual(bytes);
+    expect(fetchImpl).toHaveBeenCalledTimes(2);
+    expect(fetchImpl.mock.calls.map(([url]) => url)).toEqual([
+      "https://api.github.com/repos/NVIDIA/NemoClaw/actions/artifacts/9001/zip",
+      "https://api.github.com/repos/NVIDIA/NemoClaw/actions/artifacts/9001/zip",
+    ]);
+    expect(sleep).toHaveBeenCalledWith(2000);
+    const evidence = log.mock.calls.map(([message]) => parseArtifactReadEvidence(message));
+    expect(evidence).toEqual([
+      expect.objectContaining({
+        operation: "artifact-content-read",
+        attempt: "1",
+        status: "503",
+        outcome: "retry",
+      }),
+      expect.objectContaining({
+        operation: "artifact-content-read",
+        attempt: "2",
+        outcome: "passed-after-retry",
+      }),
+    ]);
+    expect(log.mock.calls.flat().join("\n")).not.toMatch(/secret|upstream body|Authorization/u);
+  });
+
+  it("fails after three transient responses without changing identity", async () => {
+    const fetchImpl = vi
+      .fn<(input: string, init: RequestInit) => Promise<Response>>()
+      .mockResolvedValue(new Response(null, { status: 500 }));
+    const sleep = vi.fn<(milliseconds: number) => Promise<void>>().mockResolvedValue();
+    await expect(downloadBoundArtifact(identity(), "token", { fetchImpl, sleep })).rejects.toThrow(
+      "HTTP 500",
+    );
+    expect(fetchImpl).toHaveBeenCalledTimes(3);
+    expect(sleep).toHaveBeenCalledTimes(2);
+  });
+
+  it.each([408, 429])("retries transient HTTP %i", async (status) => {
+    const bytes = archive();
+    const fetchImpl = vi
+      .fn<(input: string, init: RequestInit) => Promise<Response>>()
+      .mockResolvedValueOnce(new Response(null, { status }))
+      .mockResolvedValueOnce(response(bytes));
+    const sleep = vi.fn<(milliseconds: number) => Promise<void>>().mockResolvedValue();
+    await expect(
+      downloadBoundArtifact(identity(bytes), "token", { fetchImpl, sleep }),
+    ).resolves.toEqual(bytes);
+    expect(fetchImpl).toHaveBeenCalledTimes(2);
+  });
+
+  it("retries a transport failure", async () => {
+    const bytes = archive();
+    const fetchImpl = vi
+      .fn<(input: string, init: RequestInit) => Promise<Response>>()
+      .mockRejectedValueOnce(new Error("connection reset"))
+      .mockResolvedValueOnce(response(bytes));
+    const sleep = vi.fn<(milliseconds: number) => Promise<void>>().mockResolvedValue();
+    await expect(
+      downloadBoundArtifact(identity(bytes), "token", { fetchImpl, sleep }),
+    ).resolves.toEqual(bytes);
+    expect(fetchImpl).toHaveBeenCalledTimes(2);
+  });
+
+  it.each([401, 403, 404, 410, 422])("does not retry terminal HTTP %i", async (status) => {
+    const fetchImpl = vi
+      .fn<(input: string, init: RequestInit) => Promise<Response>>()
+      .mockResolvedValue(new Response(null, { status }));
+    await expect(downloadBoundArtifact(identity(), "token", { fetchImpl })).rejects.toThrow(
+      `HTTP ${status}`,
+    );
+    expect(fetchImpl).toHaveBeenCalledTimes(1);
+  });
+
+  it("does not retry digest mismatch", async () => {
+    const expectedBytes = archive();
+    const actualBytes = Buffer.from(expectedBytes);
+    actualBytes[0] ^= 0xff;
+    const fetchImpl = vi
+      .fn<(input: string, init: RequestInit) => Promise<Response>>()
+      .mockResolvedValue(response(actualBytes));
+    await expect(
+      downloadBoundArtifact(identity(expectedBytes), "token", { fetchImpl }),
+    ).rejects.toThrow("digest");
+    expect(fetchImpl).toHaveBeenCalledTimes(1);
+  });
+
+  it("does not retry a content-length mismatch", async () => {
+    const bytes = archive();
+    const fetchImpl = vi
+      .fn<(input: string, init: RequestInit) => Promise<Response>>()
+      .mockResolvedValue(
+        new Response(new Uint8Array(bytes), {
+          headers: { "content-length": String(bytes.length + 1) },
+        }),
+      );
+    await expect(downloadBoundArtifact(identity(bytes), "token", { fetchImpl })).rejects.toThrow(
+      "content length",
+    );
+    expect(fetchImpl).toHaveBeenCalledTimes(1);
+  });
+
+  it("stops an unbounded response stream before retaining oversized content", async () => {
+    const bytes = archive();
+    const fetchImpl = vi
+      .fn<(input: string, init: RequestInit) => Promise<Response>>()
+      .mockResolvedValue(new Response(new Uint8Array(Buffer.concat([bytes, Buffer.from("x")]))));
+    await expect(downloadBoundArtifact(identity(bytes), "token", { fetchImpl })).rejects.toThrow(
+      "content size",
+    );
+    expect(fetchImpl).toHaveBeenCalledTimes(1);
+  });
+
+  it.each([
+    [{ attempts: 0 }, "attempts must be between"],
+    [{ attempts: 4 }, "attempts must be between"],
+    [{ timeoutMs: 0 }, "timeout must be between"],
+    [{ timeoutMs: 20_001 }, "timeout must be between"],
+  ])("rejects invalid download bounds without a request", async (options, message) => {
+    const fetchImpl = vi.fn<(input: string, init: RequestInit) => Promise<Response>>();
+    await expect(
+      downloadBoundArtifact(identity(), "token", { ...options, fetchImpl }),
+    ).rejects.toThrow(message);
+    expect(fetchImpl).not.toHaveBeenCalled();
+  });
+
+  it("rejects a multiline token without exposing or sending it", async () => {
+    const fetchImpl = vi.fn<(input: string, init: RequestInit) => Promise<Response>>();
+    const token = "secret-token\nsecond-line";
+    const failure = downloadBoundArtifact(identity(), token, { fetchImpl });
+    await expect(failure).rejects.toThrow("single-line value");
+    await expect(failure).rejects.not.toThrow(/secret-token|second-line/u);
+    expect(fetchImpl).not.toHaveBeenCalled();
+  });
+
+  it("rejects malformed archives before writing a contract", () => {
+    const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-artifact-"));
+    try {
+      expect(() => materializeContractArchive(Buffer.from("not a zip"), directory)).toThrow(
+        "exactly one contract.json",
+      );
+      expect(fs.readdirSync(directory)).toEqual([]);
+    } finally {
+      fs.rmSync(directory, { force: true, recursive: true });
+    }
+  });
+
+  it("leaves contract semantics to the existing fail-closed validator", () => {
+    const directory = fs.mkdtempSync(path.join(os.tmpdir(), "nemoclaw-artifact-"));
+    try {
+      const contractPath = materializeContractArchive(archive(), directory);
+      const value = JSON.parse(fs.readFileSync(contractPath, "utf8")) as unknown;
+      expect(() => validateDcodeBaseImageContract(value, EXPECTED)).toThrow();
+    } finally {
+      fs.rmSync(directory, { force: true, recursive: true });
+    }
+  });
+});