Merge branch 'main' into codex/issue-3768-gateway-recovery-deadline

Author: cjagwani

test/e2e/fixtures/cleanup.ts

@@ -14,6 +14,12 @@ export interface CleanupResult {
 type CleanupFn = () => Promise<void> | void;
 type RedactFn = (text: string) => string;
 
+export interface CleanupHost {
+  cleanupSandbox(name: string): Promise<void>;
+  cleanupGatewayRegistration(name: string): Promise<void>;
+  cleanupForward(port: number): Promise<void>;
+}
+
 interface CleanupEntry {
   name: string;
   run: CleanupFn;
@@ -34,6 +40,22 @@ export class CleanupRegistry {
     this.entries.push({ name, run });
   }
 
+  trackSandbox(host: CleanupHost, name: string): void {
+    this.add(`destroy sandbox ${name}`, () => host.cleanupSandbox(name));
+  }
+
+  trackGateway(host: CleanupHost, name: string): void {
+    this.add(`remove gateway ${name}`, () => host.cleanupGatewayRegistration(name));
+  }
+
+  trackForward(host: CleanupHost, port: number): void {
+    this.add(`stop forward ${port}`, () => host.cleanupForward(port));
+  }
+
+  trackDisposable(name: string, dispose: CleanupFn): void {
+    this.add(name, dispose);
+  }
+
   async runAll(): Promise<CleanupResult> {
     const result: CleanupResult = { passed: [], failures: [] };
     for (const entry of [...this.entries].reverse()) {

test/e2e/fixtures/clients/host.ts

@@ -21,6 +21,8 @@ const GATEWAY_ALREADY_ABSENT =
   /gateway[^\n]*(?:does not exist|not found)|No (?:active )?gateway|No gateway metadata found/i;
 const GATEWAY_REMOVE_UNSUPPORTED =
   /unrecognized subcommand ['"]remove['"]|unknown command ['"]remove['"]/i;
+const FORWARD_ALREADY_ABSENT =
+  /no (?:active )?forward|forward[^\n]*(?:not found|not running)|forward stop[^\n]*not running/i;
 
 export class HostCliClient {
   private readonly runner: CommandRunner;
@@ -151,6 +153,15 @@ export class HostCliClient {
     assertExitZero(destroy, `cleanup gateway registration ${gatewayName}`);
   }
 
+  async cleanupForward(port: number, options: ShellProbeRunOptions = {}): Promise<void> {
+    const result = await this.command("openshell", ["forward", "stop", String(port)], {
+      ...options,
+      artifactName: options.artifactName ?? `cleanup-forward-${port}`,
+    });
+    if (result.exitCode === 0 || FORWARD_ALREADY_ABSENT.test(resultText(result))) return;
+    assertExitZero(result, `cleanup forward ${port}`);
+  }
+
   async bestEffortCleanupSandbox(
     sandboxName: string,
     options: ShellProbeRunOptions = {},

test/e2e/live/hermes-gpu-startup-integrity.ts

@@ -76,18 +76,23 @@ def parse_hash(data, label):
         text = data.decode("ascii")
     except UnicodeDecodeError:
         fail(f"{label} is not ASCII")
-    if not text.endswith("\\n"):
-        fail(f"{label} is missing its final newline")
-    lines = text.splitlines()
+    parts = text.split("\\n")
+    if len(parts) != 4 or parts[-1] != "":
+        fail(f"{label} does not contain exactly three records")
+    lines = parts[:2]
     expected_paths = (str(config_path), str(env_path))
-    if len(lines) != len(expected_paths):
-        fail(f"{label} does not contain exactly two records")
     digests = []
     for line, expected_path in zip(lines, expected_paths):
         match = re.fullmatch(r"([0-9a-f]{64})  (.+)", line)
         if match is None or match.group(2) != expected_path:
-            fail(f"{label} contains an unexpected record")
+            fail(f"{label} contains an unexpected file record")
         digests.append(match.group(1))
+    state_match = re.fullmatch(
+        r"# nemoclaw-hermes-mcp-state-v1 intended=([0-9a-f]{64}) applied=([0-9a-f]{64})",
+        parts[2],
+    )
+    if state_match is None:
+        fail(f"{label} contains an unexpected MCP state record")
     return tuple(digests)
 
 def digest(data):

test/e2e/live/sandbox-operations.test.ts

@@ -13,6 +13,7 @@ import os from "node:os";
 import path from "node:path";
 import { containsInteger42Answer } from "../../helpers/e2e-answer-assertions.ts";
 import { buildAvailabilityProbeEnv } from "../fixtures/availability-env.ts";
+import type { CleanupRegistry } from "../fixtures/cleanup.ts";
 import {
   assertExitZero as expectExitZero,
   outputContainsSandbox,
@@ -34,8 +35,6 @@ const SANDBOX_B = "e2e-sbx-b";
 const REGISTRY_FILE = path.join(process.env.HOME ?? os.homedir(), ".nemoclaw", "sandboxes.json");
 const GATEWAY_CONTAINER = "openshell-cluster-nemoclaw";
 
-type CleanupRegistry = { add(name: string, run: () => Promise<void> | void): void };
-
 async function onboardSandbox(
   host: HostCliClient,
   cleanup: CleanupRegistry,
@@ -44,7 +43,7 @@ async function onboardSandbox(
   hosted: HostedInferenceConfig,
   extraEnv: NodeJS.ProcessEnv = {},
 ): Promise<ShellProbeResult> {
-  cleanup.add(`destroy sandbox ${sandboxName}`, () => host.cleanupSandbox(sandboxName));
+  cleanup.trackSandbox(host, sandboxName);
   const result = await host.nemoclaw(
     ["onboard", "--non-interactive", "--yes", "--yes-i-accept-third-party-software"],
     {

test/e2e/support/e2e-cleanup-resources.test.ts

@@ -0,0 +1,61 @@
+// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+import { describe, expect, it } from "vitest";
+
+import { type CleanupHost, CleanupRegistry } from "../fixtures/cleanup.ts";
+
+describe("cleanup resources", () => {
+  it("tears down acquired resources in reverse order", async () => {
+    const calls: string[] = [];
+    const host: CleanupHost = {
+      cleanupSandbox: async (name) => {
+        calls.push(`sandbox:${name}`);
+      },
+      cleanupGatewayRegistration: async (name) => {
+        calls.push(`gateway:${name}`);
+      },
+      cleanupForward: async (port) => {
+        calls.push(`forward:${port}`);
+      },
+    };
+    const cleanup = new CleanupRegistry();
+    cleanup.trackGateway(host, "nemoclaw");
+    cleanup.trackSandbox(host, "e2e-resource");
+    cleanup.trackForward(host, 18789);
+
+    const result = await cleanup.runAll();
+    expect(calls).toEqual(["forward:18789", "sandbox:e2e-resource", "gateway:nemoclaw"]);
+    expect(result.failures).toEqual([]);
+  });
+
+  it("supports partial setup and runs each registration only once", async () => {
+    let calls = 0;
+    const cleanup = new CleanupRegistry();
+    cleanup.trackDisposable("close acquired server", () => {
+      calls += 1;
+    });
+
+    expect((await cleanup.runAll()).passed).toEqual(["close acquired server"]);
+    expect(await cleanup.runAll()).toEqual({ passed: [], failures: [] });
+    expect(calls).toBe(1);
+  });
+
+  it("redacts failures and continues cleanup", async () => {
+    const calls: string[] = [];
+    const cleanup = new CleanupRegistry((text) => text.replaceAll("secret", "[REDACTED]"));
+    cleanup.trackDisposable("later secret cleanup", () => {
+      calls.push("later");
+    });
+    cleanup.trackDisposable("failing secret cleanup", () => {
+      throw new Error("secret failure");
+    });
+
+    const result = await cleanup.runAll();
+    expect(calls).toEqual(["later"]);
+    expect(result).toEqual({
+      passed: ["later [REDACTED] cleanup"],
+      failures: [{ name: "failing [REDACTED] cleanup", message: "[REDACTED] failure" }],
+    });
+  });
+});

test/e2e/support/e2e-clients.test.ts

@@ -183,6 +183,34 @@ describe("E2E fixture clients", () => {
     ]);
   });
 
+  it.each([
+    "No active forward",
+    "forward 18789 not found",
+    "forward stop failed: not running",
+  ])("host client accepts canonical already-absent forward output: %s", async (stderr) => {
+    const runner = new FakeRunner();
+    runner.enqueue({ exitCode: 1, stderr });
+    const host = new HostCliClient(runner, { cliPath: "nemoclaw" });
+
+    await host.cleanupForward(18789);
+
+    expect(runner.calls.map((call) => call.args)).toEqual([["forward", "stop", "18789"]]);
+  });
+
+  it.each([
+    "permission denied",
+    "daemon not running",
+    "some unrelated error: not running",
+  ])("host client surfaces unexpected forward cleanup failure: %s", async (stderr) => {
+    const runner = new FakeRunner();
+    runner.enqueue({ exitCode: 1, stderr });
+    const host = new HostCliClient(runner, { cliPath: "nemoclaw" });
+
+    await expect(host.cleanupForward(18789)).rejects.toThrow(
+      `cleanup forward 18789 failed: ${stderr}`,
+    );
+  });
+
   it("host client does not hide a current gateway remove failure behind the legacy verb", async () => {
     const runner = new FakeRunner();
     runner.enqueue({ exitCode: 1, stderr: "permission denied" });

test/e2e/support/hermes-gpu-startup-integrity.test.ts

@@ -22,6 +22,7 @@ interface IntegrityFixture {
 }
 
 const roots: string[] = [];
+const MCP_STATE_RECORD = `# nemoclaw-hermes-mcp-state-v1 intended=${"1".repeat(64)} applied=${"2".repeat(64)}`;
 
 afterEach(() => {
   for (const root of roots.splice(0)) fs.rmSync(root, { recursive: true, force: true });
@@ -38,7 +39,18 @@ function writeHash(
   envPath: string,
   env: string,
 ): void {
-  fs.writeFileSync(hashPath, `${digest(config)}  ${configPath}\n${digest(env)}  ${envPath}\n`);
+  fs.writeFileSync(
+    hashPath,
+    `${digest(config)}  ${configPath}\n${digest(env)}  ${envPath}\n${MCP_STATE_RECORD}\n`,
+  );
+}
+
+function readHashRecords(hashPath: string): string[] {
+  return fs.readFileSync(hashPath, "utf-8").split("\n").slice(0, -1);
+}
+
+function writeHashRecords(hashPath: string, records: readonly string[]): void {
+  fs.writeFileSync(hashPath, `${records.join("\n")}\n`);
 }
 
 function createFixture(): IntegrityFixture {
@@ -96,7 +108,7 @@ function runProof(fixture: IntegrityFixture, extraEnv: NodeJS.ProcessEnv = {}) {
 }
 
 describe("Hermes managed startup integrity proof", () => {
-  it("accepts a current compatibility hash and one generated API key beyond the strict base", () => {
+  it("accepts canonical file and MCP state records with one generated API key beyond the strict base (#6427)", () => {
     const fixture = createFixture();
     const rawStrictCheck = spawnSync("sha256sum", ["-c", fixture.strictHashPath, "--status"], {
       encoding: "utf-8",
@@ -111,6 +123,68 @@ describe("Hermes managed startup integrity proof", () => {
     expect(proof.stdout).toBe("OK\n");
   });
 
+  it("rejects a missing Hermes MCP state record (#6427)", () => {
+    const fixture = createFixture();
+    const [configRecord, envRecord] = readHashRecords(fixture.compatHashPath);
+    writeHashRecords(fixture.compatHashPath, [configRecord!, envRecord!]);
+
+    const proof = runProof(fixture);
+    expect(proof.status).not.toBe(0);
+    expect(proof.stderr).toContain(
+      "Hermes compatibility hash does not contain exactly three records",
+    );
+  });
+
+  it("rejects a malformed Hermes MCP state record (#6427)", () => {
+    const fixture = createFixture();
+    const [configRecord, envRecord] = readHashRecords(fixture.compatHashPath);
+    writeHashRecords(fixture.compatHashPath, [
+      configRecord!,
+      envRecord!,
+      `# nemoclaw-hermes-mcp-state-v1 intended=${"1".repeat(64)} applied=invalid`,
+    ]);
+
+    const proof = runProof(fixture);
+    expect(proof.status).not.toBe(0);
+    expect(proof.stderr).toContain(
+      "Hermes compatibility hash contains an unexpected MCP state record",
+    );
+  });
+
+  it("rejects duplicate Hermes MCP state records (#6427)", () => {
+    const fixture = createFixture();
+    const records = readHashRecords(fixture.compatHashPath);
+    writeHashRecords(fixture.compatHashPath, [...records, MCP_STATE_RECORD]);
+
+    const proof = runProof(fixture);
+    expect(proof.status).not.toBe(0);
+    expect(proof.stderr).toContain(
+      "Hermes compatibility hash does not contain exactly three records",
+    );
+  });
+
+  it("rejects a reordered Hermes MCP state record (#6427)", () => {
+    const fixture = createFixture();
+    const [configRecord, envRecord, stateRecord] = readHashRecords(fixture.compatHashPath);
+    writeHashRecords(fixture.compatHashPath, [stateRecord!, configRecord!, envRecord!]);
+
+    const proof = runProof(fixture);
+    expect(proof.status).not.toBe(0);
+    expect(proof.stderr).toContain("Hermes compatibility hash contains an unexpected file record");
+  });
+
+  it("rejects unexpected records after the Hermes MCP state record (#6427)", () => {
+    const fixture = createFixture();
+    const records = readHashRecords(fixture.compatHashPath);
+    writeHashRecords(fixture.compatHashPath, [...records, "unexpected"]);
+
+    const proof = runProof(fixture);
+    expect(proof.status).not.toBe(0);
+    expect(proof.stderr).toContain(
+      "Hermes compatibility hash does not contain exactly three records",
+    );
+  });
+
   it("rejects non-key environment drift even when the compatibility hash accepts it", () => {
     const fixture = createFixture();
     const config = fs.readFileSync(fixture.configPath, "utf-8");