fix(onboard): accept Hermes startup CMD in root helper (#9511)

<!-- markdownlint-disable MD041 -->
## Summary
Hermes managed startup could reach the root helper with Docker-appended
image `CMD` arguments, but the helper rejected that valid process shape
before recording its durable fence. This change accepts trailing
arguments only after the fixed `nemoclaw-start` position while
preserving complete process identity binding.

## Related Issue
Fixes #9485

## Changes
- Authenticate the exact `/usr/local/bin/nemoclaw-start` path when it is
direct or immediately interpreted by an allowed Bash path, including
Docker-appended trailing `CMD` arguments.
- Continue hashing and fencing the complete argument vector, and reject
bare, prefixed, or reordered startup-script forms.
- Model Docker named-volume state in the mutation harness and verify
recovery when the durable-volume acquire succeeds but its response is
lost.
- Documentation writer review found no documentation change is needed
because this restores the existing managed Hermes topology without
changing commands, configuration, defaults, or user workflow.

## Type of Change

- [x] Code change (feature, bug fix, or refactor)
- [ ] Code change with doc updates
- [ ] Doc only (prose changes, no code sample modifications)
- [ ] Doc only (includes code sample changes)

## Quality Gates
<!-- Check one tests line. Check other lines when applicable. Add every
requested justification or approval reference. -->
- [x] Tests added or updated for changed behavior
- [ ] Existing tests cover changed behavior — justification:
- [ ] Tests not applicable — justification:
- [x] Sensitive paths changed (security, policy, credentials, preflight,
onboarding, inference, runner, sandbox, or messaging)
- [x] Sensitive-path review completed or maintainer-approved waiver
recorded — reviewer/approval link/justification: Complete diff reviewed
for exact process identity, parent and UID checks, full argument
hashing, durable-volume recovery, and fail-closed negative forms.
- [ ] Non-success, skipped, or missing CI check accepted by maintainer —
check name, approval link, and follow-up issue:

## DGX Station Hardware Evidence
<!-- Required only when scripts/prepare-dgx-station-host.sh changes.
Maintainers must review the linked evidence before approving or merging.
This is human-reviewed evidence, not authenticated hardware provenance.
Exceptional bypasses use existing repository governance and must be
documented on the PR. -->
- [ ] Tested on DGX Station
- Tested commit:
- Station profile/scenario:
- Result:
- Supporting evidence:

## Verification
<!-- Check each applicable item only when supported by the requested
evidence. Run targeted tests once per relevant change set and rerun
after later edits or hook autofixes that can affect the tested behavior.
Do not rerun hook-covered checks. -->
- [x] PR description includes a `Signed-off-by:` line and every commit
appears as `Verified` in GitHub
- [x] Normal `pre-commit`, `commit-msg`, and `pre-push` hooks passed, or
`npm run validate:pr` passed after refreshing `origin/main` when hooks
were skipped or unavailable
- [x] Targeted behavior tests pass for the current change set, or tests
are marked not applicable above — command/result or justification:
`vitest` root-helper integration, 11/11 passed; Docker, Podman, and
persisted-engine lifecycle suite, 83/83 passed; `npm run build:cli` and
`npm run typecheck:cli` passed.
- [ ] Applicable broad gate passed — `npm test` for broad
runtime/test-harness changes; `npm run check` for repo-wide
validation/coverage changes — command/result:
- [x] Quality Gates section completed with required justifications or
waivers
- [x] No secrets, API keys, or credentials committed
- [ ] `npm run docs` builds without warnings (doc changes only)
- [ ] Doc pages follow the [style
guide](https://github.com/NVIDIA/NemoClaw/blob/main/docs/CONTRIBUTING.md)
(doc changes only)
- [ ] New doc pages include SPDX header and frontmatter (new pages only)

---
<!-- DCO sign-off is required in this PR description, and every commit
must appear as Verified in GitHub. Run: git config user.name && git
config user.email -->
Signed-off-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com>


<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->

## Summary by CodeRabbit

- **Bug Fixes**
- Improved recognition of supported `nemoclaw-start` launch commands,
including Docker-appended arguments and approved Bash invocation paths.
- Tightened validation to reject unsupported or reordered command
formats.
- Improved recovery handling when durable-volume state acquisition
responses are lost.

- **Tests**
- Expanded coverage for bind-mounted and managed-volume state
configurations.
- Added validation across multiple runtime command layouts and volume
metadata scenarios.

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

Co-authored-by: Rebecca Sliter <571084+rsliter@users.noreply.github.com>

Author: rsliter

scripts/runtime-state-mutation-control.py

@@ -105,6 +105,7 @@
 STARTUP_RETRY_ACK_PROTOCOL = "nemoclaw-runtime-state-mutation-retry-ack-v1"
 OPENSHELL_ARGV0 = b"/opt/openshell/bin/openshell-sandbox"
 NEMOCLAW_START_PATH = b"/usr/local/bin/nemoclaw-start"
+BASH_ARGV0 = (b"bash", b"/bin/bash", b"/usr/bin/bash")
 HERMES_GATEWAY_PATHS = (b"/usr/local/bin/hermes", b"/usr/local/bin/hermes.real")
 HERMES_INTERNAL_PORT = 18642
 HERMES_HEALTH_PATH = "/health"
@@ -2402,10 +2403,12 @@ def _is_openshell_supervisor(process: ProcessIdentity) -> bool:
 
 
 def _is_nemoclaw_start(process: ProcessIdentity, sandbox_uid: int) -> bool:
-    direct = process.command == (NEMOCLAW_START_PATH,)
+    # Docker appends image CMD arguments after ENTRYPOINT. Authenticate the
+    # fixed startup-script position, then bind the complete argv to the fence.
+    direct = bool(process.command) and process.command[0] == NEMOCLAW_START_PATH
     interpreted = bool(
-        len(process.command) == 2
-        and process.command[0].rsplit(b"/", 1)[-1] == b"bash"
+        len(process.command) >= 2
+        and process.command[0] in BASH_ARGV0
         and process.command[1] == NEMOCLAW_START_PATH
     )
     return bool(

src/lib/onboard/runtime-provider/docker-state-mutation.test.ts

@@ -534,8 +534,13 @@ describe("Docker state mutation owner", () => {
     expect(runtime.lifecycleStore.listUnfinished()[0]?.phase).toBe("fence-established");
   });
 
-  it("converges when an orphan acquire writes its marker before recovery", () => {
-    const runtime = harness({ loseAcquireResponseOnce: true });
+  it("recovers a durable-volume fence when acquire succeeds after its response is lost (#9485)", () => {
+    const runtime = harness({ loseAcquireResponseOnce: true, stateMountType: "volume" });
+    expect(runtime.state).toMatchObject({
+      mountDriver: "local",
+      mountName: "nemoclaw-hermes-alpha-state",
+      mountType: "volume",
+    });
 
     expect(() => runtime.owner.acquire({ ...runtime.context, plan: plan() })).toThrow(
       "root helper acquire did not complete successfully",

test/helpers/docker-state-mutation-harness.ts

@@ -139,11 +139,15 @@ export interface DockerStateMutationHarnessOptions {
   readonly lifecycleGeneration?: string;
   readonly loseAcquireResponseOnce?: boolean;
   readonly loseReleaseResponseOnce?: boolean;
+  readonly stateMountType?: "bind" | "volume";
 }
 
 export interface DockerStateMutationHarnessState {
+  mountDriver: string | null;
+  mountName: string | null;
   runtimePid: number;
   mountSource: string;
+  mountType: "bind" | "volume";
   sandboxId: string;
   pidMode: string;
   privileged: boolean;
@@ -156,9 +160,16 @@ function createContainerStateMutationHarness(
 ) {
   const lifecycleGeneration =
     options.lifecycleGeneration ?? DOCKER_STATE_MUTATION_LIFECYCLE_GENERATION;
+  const stateMountType = options.stateMountType ?? "bind";
+  const usesManagedVolume = stateMountType === "volume";
   const state: DockerStateMutationHarnessState = {
+    mountDriver: usesManagedVolume ? "local" : null,
+    mountName: usesManagedVolume ? "nemoclaw-hermes-alpha-state" : null,
     runtimePid: 4812,
-    mountSource: "/var/lib/openshell/alpha/hermes",
+    mountSource: usesManagedVolume
+      ? "/var/lib/docker/volumes/nemoclaw-hermes-alpha-state/_data"
+      : "/var/lib/openshell/alpha/hermes",
+    mountType: stateMountType,
     sandboxId: SANDBOX_ID,
     pidMode: "",
     privileged: false,
@@ -229,9 +240,11 @@ function createContainerStateMutationHarness(
           state.privileged,
           [
             {
-              Type: "bind",
+              Type: state.mountType,
               Source: state.mountSource,
+              ...(state.mountName === null ? {} : { Name: state.mountName }),
               Destination: DOCKER_STATE_MUTATION_STATE_ROOT,
+              ...(state.mountDriver === null ? {} : { Driver: state.mountDriver }),
               Mode: "",
               RW: true,
               Propagation: "rprivate",

test/runtime-state-mutation-control.test.ts

@@ -187,15 +187,14 @@ def process(pid, state, parent, start, uid, command, inode):
 
 root_uid = control.ROOT_UID
 pid1 = process(1, "S", 0, "100", root_uid, (control.OPENSHELL_ARGV0,), 101)
-start = process(
-    10,
-    "S",
-    1,
-    "200",
-    1001,
-    (b"/bin/bash", control.NEMOCLAW_START_PATH),
-    110,
-)
+def start_process(pid, command):
+    return process(pid, "S", 1, str(190 + pid), 1001, command, 100 + pid)
+
+start = start_process(10, (b"/bin/bash", control.NEMOCLAW_START_PATH, b"/bin/bash"))
+prefixed_start = start_process(11, (b"/bin/bash", b"--noprofile", control.NEMOCLAW_START_PATH))
+reordered_start = start_process(12, (b"/bin/bash", b"/bin/bash", control.NEMOCLAW_START_PATH))
+bare_direct_start = start_process(13, (b"nemoclaw-start", b"/bin/bash"))
+bare_interpreted_start = start_process(14, (b"/bin/bash", b"nemoclaw-start", b"/bin/bash"))
 gateway = process(
     77,
     "S",
@@ -483,6 +482,11 @@ control._supported_writer_uids = lambda: (1000, 1001)
 control._sandbox_uid = lambda: 1001
 control._capture_process = lambda pid: {1: pid1, 10: start}.get(pid)
 control._capture_writer_processes = lambda _uids: (start, gateway)
+results["managed_start_with_cmd"] = control._is_nemoclaw_start(start, 1001)
+results["prefixed_start"] = control._is_nemoclaw_start(prefixed_start, 1001)
+results["reordered_start"] = control._is_nemoclaw_start(reordered_start, 1001)
+results["bare_direct_start"] = control._is_nemoclaw_start(bare_direct_start, 1001)
+results["bare_interpreted_start"] = control._is_nemoclaw_start(bare_interpreted_start, 1001)
 real_readlink = os.readlink
 os.readlink = lambda selected: "mnt:[401]" if selected == control.MOUNT_NAMESPACE_PATH else real_readlink(selected)
 try:
@@ -1273,10 +1277,15 @@ describe("runtime state mutation controller", () => {
     });
   });
 
-  it("holds exact OpenShell and entrypoint identities through recovery (#7744)", () => {
+  it("holds exact OpenShell and managed-image entrypoint identities through recovery (#9485)", () => {
     expect(harnessResult).toMatchObject({
       acquire: "fenced",
       assert: "fenced",
+      managed_start_with_cmd: true,
+      prefixed_start: false,
+      reordered_start: false,
+      bare_direct_start: false,
+      bare_interpreted_start: false,
       acquire_fence: {
         supervisor: { pid: 1, startIdentity: "100" },
         start: { pid: 10, startIdentity: "200", parentPid: 1 },