Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions .github/workflows/platform-vitest-main.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -251,7 +251,7 @@ jobs:
"@
Invoke-WslScript -Distro $env:WSL_DISTRO -User $env:WSL_TEST_USER -Script $script

- name: Run full Vitest suite in WSL
- name: Run Vitest suite in WSL
shell: powershell
run: |
. "$env:TRUSTED_WSL_HELPER"
Expand All @@ -263,7 +263,8 @@ jobs:
cd $workdir
export NEMOCLAW_EXEC_TIMEOUT=60000
export NEMOCLAW_TEST_TIMEOUT=60000
npx vitest run --testTimeout 60000 --shard='${{ matrix.shard }}/4'
npx vitest run --testTimeout 60000 --shard='${{ matrix.shard }}/4' \
--exclude test/jetson-device-group-bootstrap.test.ts
"@
Invoke-WslScript -Distro $env:WSL_DISTRO -User $env:WSL_TEST_USER -Script $script

Expand Down
7 changes: 6 additions & 1 deletion Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -564,6 +564,7 @@
COPY scripts/lib/entrypoint-env-wrapper.sh /usr/local/lib/nemoclaw/entrypoint-env-wrapper.sh
COPY scripts/lib/gateway-supervisor.sh /usr/local/lib/nemoclaw/gateway-supervisor.sh
COPY scripts/lib/sandbox-rlimits.sh /usr/local/lib/nemoclaw/sandbox-rlimits.sh
COPY scripts/jetson-device-group-bootstrap.sh /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh
COPY scripts/lib/openclaw_device_approval_policy.py /usr/local/lib/nemoclaw/openclaw_device_approval_policy.py
COPY scripts/lib/clean_runtime_shell_env_shim.py /usr/local/lib/nemoclaw/clean_runtime_shell_env_shim.py
COPY scripts/lib/normalize_mutable_config_perms.py /usr/local/lib/nemoclaw/normalize_mutable_config_perms.py
Expand Down Expand Up @@ -1570,12 +1571,14 @@
&& chmod -R a+rX /src/lib/messaging \
&& chown root:root /usr/local/bin/nemoclaw-gateway-control \
/usr/local/lib/nemoclaw/gateway-supervisor.sh \
/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh \
/usr/local/lib/nemoclaw/state-dir-guard.py \
/usr/local/share/nemoclaw/state-lock-plan.json \
/usr/local/lib/nemoclaw/openclaw-config-guard.py \
/usr/local/lib/nemoclaw/managed-gateway-control.py \
&& chmod 700 /usr/local/bin/nemoclaw-gateway-control \
&& chmod 500 /usr/local/lib/nemoclaw/state-dir-guard.py \
&& chmod 500 /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh \
/usr/local/lib/nemoclaw/state-dir-guard.py \
/usr/local/lib/nemoclaw/openclaw-config-guard.py \
/usr/local/lib/nemoclaw/managed-gateway-control.py \
&& chmod 444 /usr/local/share/nemoclaw/state-lock-plan.json \
Expand Down Expand Up @@ -1660,10 +1663,10 @@
# CHAT_UI_URL is a non-loopback address (Brev Launchable, remote deployments)
# since terminal-based pairing is impossible in those contexts.
# Default: "0" (device auth enabled for local deployments — secure by default).
ARG NEMOCLAW_DISABLE_DEVICE_AUTH=0

Check warning on line 1666 in Dockerfile

View workflow job for this annotation

GitHub Actions / PR build and direct managed startup (OpenClaw)

Sensitive data should not be used in the ARG or ENV commands

SecretsUsedInArgOrEnv: Do not use ARG or ENV instructions for sensitive data (ARG "NEMOCLAW_DISABLE_DEVICE_AUTH") More info: https://docs.docker.com/go/dockerfile/rule/secrets-used-in-arg-or-env/

Check warning on line 1666 in Dockerfile

View workflow job for this annotation

GitHub Actions / PR build and direct managed startup (OpenClaw)

Sensitive data should not be used in the ARG or ENV commands

SecretsUsedInArgOrEnv: Do not use ARG or ENV instructions for sensitive data (ARG "NEMOCLAW_DISABLE_DEVICE_AUTH") More info: https://docs.docker.com/go/dockerfile/rule/secrets-used-in-arg-or-env/
# Internal audit provenance for the opt-out above. Standard onboarding rewrites
# this to managed-onboard; direct image builders retain operator provenance.
ARG NEMOCLAW_DEVICE_AUTH_OPT_OUT_SOURCE=operator

Check warning on line 1669 in Dockerfile

View workflow job for this annotation

GitHub Actions / PR build and direct managed startup (OpenClaw)

Sensitive data should not be used in the ARG or ENV commands

SecretsUsedInArgOrEnv: Do not use ARG or ENV instructions for sensitive data (ARG "NEMOCLAW_DEVICE_AUTH_OPT_OUT_SOURCE") More info: https://docs.docker.com/go/dockerfile/rule/secrets-used-in-arg-or-env/

Check warning on line 1669 in Dockerfile

View workflow job for this annotation

GitHub Actions / PR build and direct managed startup (OpenClaw)

Sensitive data should not be used in the ARG or ENV commands

SecretsUsedInArgOrEnv: Do not use ARG or ENV instructions for sensitive data (ARG "NEMOCLAW_DEVICE_AUTH_OPT_OUT_SOURCE") More info: https://docs.docker.com/go/dockerfile/rule/secrets-used-in-arg-or-env/
# Compatibility build arg for older custom Dockerfiles and rebuild tooling.
# NemoClaw-managed images intentionally do not consume it; gateway auth tokens
# are generated at container startup and are never baked into image layers.
Expand Down Expand Up @@ -1697,7 +1700,7 @@
# NEMOCLAW_MESSAGING_PLAN_B64 intentionally remains ARG-only: Docker exposes it
# to build RUN processes without retaining the full plan in the final image env.
# Direct ARG interpolation into inline source is a code injection vector (C-2).
ENV NEMOCLAW_MODEL=${NEMOCLAW_MODEL} \

Check warning on line 1703 in Dockerfile

View workflow job for this annotation

GitHub Actions / PR build and direct managed startup (OpenClaw)

Sensitive data should not be used in the ARG or ENV commands

SecretsUsedInArgOrEnv: Do not use ARG or ENV instructions for sensitive data (ENV "NEMOCLAW_DEVICE_AUTH_OPT_OUT_SOURCE") More info: https://docs.docker.com/go/dockerfile/rule/secrets-used-in-arg-or-env/

Check warning on line 1703 in Dockerfile

View workflow job for this annotation

GitHub Actions / PR build and direct managed startup (OpenClaw)

Sensitive data should not be used in the ARG or ENV commands

SecretsUsedInArgOrEnv: Do not use ARG or ENV instructions for sensitive data (ENV "NEMOCLAW_DISABLE_DEVICE_AUTH") More info: https://docs.docker.com/go/dockerfile/rule/secrets-used-in-arg-or-env/

Check warning on line 1703 in Dockerfile

View workflow job for this annotation

GitHub Actions / PR build and direct managed startup (OpenClaw)

Sensitive data should not be used in the ARG or ENV commands

SecretsUsedInArgOrEnv: Do not use ARG or ENV instructions for sensitive data (ENV "NEMOCLAW_DEVICE_AUTH_OPT_OUT_SOURCE") More info: https://docs.docker.com/go/dockerfile/rule/secrets-used-in-arg-or-env/

Check warning on line 1703 in Dockerfile

View workflow job for this annotation

GitHub Actions / PR build and direct managed startup (OpenClaw)

Sensitive data should not be used in the ARG or ENV commands

SecretsUsedInArgOrEnv: Do not use ARG or ENV instructions for sensitive data (ENV "NEMOCLAW_DISABLE_DEVICE_AUTH") More info: https://docs.docker.com/go/dockerfile/rule/secrets-used-in-arg-or-env/
NEMOCLAW_INFERENCE_PROVIDER_ID=${NEMOCLAW_INFERENCE_PROVIDER_ID} \
NEMOCLAW_UPSTREAM_PROVIDER=${NEMOCLAW_UPSTREAM_PROVIDER} \
NEMOCLAW_PRIMARY_MODEL_REF=${NEMOCLAW_PRIMARY_MODEL_REF} \
Expand Down Expand Up @@ -2323,6 +2326,8 @@
&& check_metadata /usr/local/bin/nemoclaw-managed-bootstrap 'root:root:755' \
&& test ! -L /usr/local/lib/nemoclaw/managed-bootstrap-trampoline.sh \
&& check_metadata /usr/local/lib/nemoclaw/managed-bootstrap-trampoline.sh 'root:root:444' \
&& test ! -L /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh \
&& check_metadata /usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh 'root:root:500' \
&& check_metadata /usr/local/bin/nemoclaw-gateway-control 'root:root:700' \
&& check_metadata /usr/local/lib/nemoclaw/state-dir-guard.py 'root:root:500' \
&& check_metadata /usr/local/share/nemoclaw/state-lock-plan.json 'root:root:444' \
Expand Down
28 changes: 26 additions & 2 deletions docs/reference/troubleshooting.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -3006,11 +3006,35 @@ The path creates the sandbox and then recreates the OpenShell-managed Docker con
`NEMOCLAW_DOCKER_GPU_PATCH=0` is ignored because this runtime requires the compatibility patch for GPU passthrough, and onboarding logs a warning when it is set.
To skip GPU passthrough entirely, rerun with `--no-gpu` or set `NEMOCLAW_SANDBOX_GPU=0`.

<AgentOnly variant="openclaw">

#### Jetson and Tegra compatibility default

Automatic GPU onboarding uses the compatibility path directly; it does not make a native attempt first.
The path recreates the OpenShell-managed Docker container with NVIDIA GPU flags and propagates eligible host group IDs for the supported Jetson GPU device nodes.
Use `NEMOCLAW_DOCKER_GPU_PATCH=0` only for troubleshooting because it bypasses that group propagation and CUDA may not initialize.
The compatibility path recreates the OpenShell-managed Docker container with NVIDIA GPU flags.
For OpenClaw, both that compatibility recreation and a managed-bootstrap replacement on the native route propagate eligible host group IDs for the supported Jetson GPU device nodes.
Docker passes those group IDs through `--group-add` in both replacements.
OpenShell calls `initgroups()` before it starts processes as the sandbox account.
That call reloads supplementary groups from the container group database and can replace the inherited Docker memberships.

Before either replacement starts the OpenShell supervisor, NemoClaw runs a root-owned helper from the OpenClaw sandbox image.
The helper validates the complete detected group-ID list before it changes the sandbox account's group database.
It then verifies the resulting memberships and can hand off only to `/opt/openshell/bin/openshell-sandbox`.
If validation or membership verification fails, the replacement container stops.

The initial filesystem policy adds Jetson paths only when `/dev/nvmap` is an existing, non-symlink character device.
It grants read-only access to the injected NVIDIA libraries under `/opt/nvidia`.
It grants read-write access only to existing, non-symlink character devices from the eligible Tegra GPU path list.
Generic GPU and CPU-only policies do not receive these Jetson entries.

`NEMOCLAW_DOCKER_GPU_PATCH=0` selects the native route instead of the default Jetson compatibility path.
Use it only for troubleshooting because preserving the device groups does not establish that the native route initializes CUDA.

These controls address NVIDIA library visibility, eligible device access, and supplementary group preservation.
They do not establish successful `cuInit(0)` on IGX Orin with JetPack 6.2.
Use `NEMOCLAW_SANDBOX_GPU=0` for CPU-only sandbox behavior while [issue #7610](https://github.com/NVIDIA/NemoClaw/issues/7610) remains open.

</AgentOnly>

#### Common compatibility-path recovery

Expand Down
60 changes: 60 additions & 0 deletions scripts/jetson-device-group-bootstrap.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
#!/usr/bin/env bash
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

# Compatibility bridge for #7610. Remove when the minimum supported OpenShell
# release natively preserves Jetson device groups across the sandbox-user handoff.

set -euo pipefail

fail() {
printf 'Jetson device-group bootstrap: %s\n' "$*" >&2
exit 1
}

[ "$(/usr/bin/id -u)" -eq 0 ] || fail "must run as root"
[ "${1:-}" = "--device-group-gids" ] || fail "device group argument is missing"
group_gids="${2:-}"
[ "${3:-}" = "--" ] || fail "supervisor delimiter is missing"
shift 3
[ "${1:-}" = "/opt/openshell/bin/openshell-sandbox" ] \
|| fail "OpenShell supervisor entrypoint is invalid"
/usr/bin/id sandbox >/dev/null 2>&1 || fail "sandbox user is missing"
if [ ! -f /etc/group ] || [ -L /etc/group ]; then
fail "container group database is invalid"
fi

IFS=',' read -r -a gids <<<"$group_gids"
if [ "${#gids[@]}" -eq 0 ] || [ "${#gids[@]}" -gt 16 ]; then
fail "device group count is invalid"
fi

declare -A seen=()
for gid in "${gids[@]}"; do
[[ "$gid" =~ ^[1-9][0-9]{0,9}$ ]] || fail "device group ID is invalid"
[ "$gid" -le 2147483647 ] || fail "device group ID is out of range"
[ -z "${seen[$gid]:-}" ] || fail "device group ID is duplicated"
seen[$gid]=1
done

for gid in "${gids[@]}"; do
group_record="$(/usr/bin/getent group "$gid" || true)"
if [ -z "$group_record" ]; then
group_name="nemoclaw_gpu_$gid"
/usr/sbin/groupadd --gid "$gid" "$group_name"
else
IFS=':' read -r group_name _ resolved_gid _ <<<"$group_record"
if [ -z "$group_name" ] || [ "$resolved_gid" != "$gid" ]; then
fail "device group record is invalid"
fi
fi
/usr/sbin/usermod --append --groups "$group_name" sandbox
done

sandbox_groups=" $(/usr/bin/id -G sandbox) "
for gid in "${gids[@]}"; do
[[ "$sandbox_groups" == *" $gid "* ]] \
|| fail "sandbox membership verification failed"
done

exec "$@"
38 changes: 37 additions & 1 deletion src/lib/onboard/docker-gpu-jetson-groups.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,43 @@ import fs from "node:fs";

import { afterEach, describe, expect, it, vi } from "vitest";

import { detectTegraDeviceGroupGids } from "./docker-gpu-jetson-groups";
import { detectTegraDeviceGroupGids, detectTegraGpuDevicePaths } from "./docker-gpu-jetson-groups";

describe("detectTegraGpuDevicePaths", () => {
it("returns existing non-symlink character devices when nvmap anchors detection (#7610)", () => {
const paths = ["/dev/nvmap", "/dev/nvhost-gpu", "/dev/nvgpu/igpu0/link"];

expect(
detectTegraGpuDevicePaths({
listDevicePaths: () => paths,
statDevicePath: (devicePath) =>
devicePath === "/dev/nvmap"
? { isCharacterDevice: true, isSymbolicLink: false }
: devicePath === "/dev/nvgpu/igpu0/link"
? { isCharacterDevice: true, isSymbolicLink: true }
: null,
}),
).toEqual(["/dev/nvmap"]);
});

it("returns no paths when nvmap is missing, a symlink, or not a character device (#7610)", () => {
for (const nvmapAccess of [
null,
{ isCharacterDevice: true, isSymbolicLink: true },
{ isCharacterDevice: false, isSymbolicLink: false },
]) {
expect(
detectTegraGpuDevicePaths({
listDevicePaths: () => ["/dev/nvmap", "/dev/dri/renderD128"],
statDevicePath: (devicePath) =>
devicePath === "/dev/nvmap"
? nvmapAccess
: { isCharacterDevice: true, isSymbolicLink: false },
}),
).toEqual([]);
}
});
});

describe("detectTegraDeviceGroupGids", () => {
afterEach(() => {
Expand Down
39 changes: 39 additions & 0 deletions src/lib/onboard/docker-gpu-jetson-groups.ts
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ const TEGRA_GPU_DEVICE_NODES = [
"/dev/nvgpu/igpu0/as",
"/dev/nvgpu/igpu0/prof",
] as const;
const NVMAP_DEVICE = "/dev/nvmap";
const READ_WRITE_PERMISSION_BITS = 0o6;
const MAX_DOCKER_SUPPLEMENTARY_GID = 2_147_483_647;

Expand All @@ -24,6 +25,11 @@ type DeviceGroupAccess = {
mode: number;
};

type DevicePathAccess = {
isCharacterDevice: boolean;
isSymbolicLink: boolean;
};

/**
* Find real DRI render character devices without following symlinks or
* scanning other DRI device families.
Expand Down Expand Up @@ -51,6 +57,39 @@ function listTegraGpuDevicePaths(): string[] {
return [...TEGRA_GPU_DEVICE_NODES, ...discoverTegraRenderDevicePaths()];
}

/**
* Return the fixed Jetson GPU device paths that are safe to add to an
* OpenShell filesystem policy. `/dev/nvmap` anchors detection so a host with
* only an unrelated DRI render device does not receive Jetson grants.
*/
export function detectTegraGpuDevicePaths(
deps: {
statDevicePath?: (path: string) => DevicePathAccess | null;
listDevicePaths?: () => string[];
} = {},
): string[] {
const devicePaths = deps.listDevicePaths?.() ?? listTegraGpuDevicePaths();
const statDevicePath =
deps.statDevicePath ??
((devicePath: string): DevicePathAccess | null => {
try {
const stat = fs.lstatSync(devicePath);
return {
isCharacterDevice: stat.isCharacterDevice(),
isSymbolicLink: stat.isSymbolicLink(),
};
} catch {
return null;
}
});

const detectedPaths = devicePaths.filter((devicePath) => {
const access = statDevicePath(devicePath);
return access?.isCharacterDevice === true && access.isSymbolicLink === false;
});
return detectedPaths.includes(NVMAP_DEVICE) ? detectedPaths : [];
}

/**
* Source-of-truth boundary for Jetson/Tegra supplementary device groups:
*
Expand Down
45 changes: 38 additions & 7 deletions src/lib/onboard/docker-gpu-patch-clone.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,9 @@ import { openshellSandboxCommandEnvValue } from "./docker-startup-command-env";

const OPENSHELL_SANDBOX_COMMAND_ENV = "OPENSHELL_SANDBOX_COMMAND";
const OPENSHELL_SANDBOX_ENTRYPOINT = "/opt/openshell/bin/openshell-sandbox";
export const JETSON_DEVICE_GROUP_BOOTSTRAP =
"/usr/local/lib/nemoclaw/jetson-device-group-bootstrap.sh";
const MAX_JETSON_DEVICE_GROUPS = 16;
const OPENSHELL_V0_0_99_WORKDIR_COMMAND = ["--workdir", "/sandbox"] as const;
const OPENSHELL_OCI_IMAGE_USER_ENV = "OPENSHELL_OCI_IMAGE_USER";
const OPENSHELL_SANDBOX_UID_ENV = "OPENSHELL_SANDBOX_UID";
Expand Down Expand Up @@ -465,6 +468,20 @@ export function buildDockerGpuCloneRunArgs(
}
const args: string[] = ["--name", containerName, ...mode.args];
const gpuAugment = mode.kind !== "startup-command";
const extraGroupGids = (options.extraGroupGids ?? []).map((gid) => String(gid).trim());
if (
extraGroupGids.length > MAX_JETSON_DEVICE_GROUPS ||
new Set(extraGroupGids).size !== extraGroupGids.length ||
extraGroupGids.some((gid) => {
if (!/^[1-9][0-9]*$/u.test(gid)) return true;
const parsed = Number(gid);
return !Number.isSafeInteger(parsed) || parsed > 2_147_483_647;
})
) {
throw new Error("Docker clone received invalid or excessive supplementary group IDs.");
}
const preserveJetsonGroups =
options.preserveJetsonDeviceGroupMembership === true && extraGroupGids.length > 0;

// Startup-command recreation must retain OpenShell's native CDI attachment.
if (!gpuAugment) {
Expand Down Expand Up @@ -546,11 +563,10 @@ export function buildDockerGpuCloneRunArgs(
for (const hostEntry of stringArray(host.ExtraHosts)) args.push("--add-host", hostEntry);
const groupAdds = new Set(stringArray(host.GroupAdd));
for (const group of groupAdds) args.push("--group-add", group);
for (const gid of options.extraGroupGids ?? []) {
const normalized = String(gid).trim();
if (normalized && !groupAdds.has(normalized)) {
groupAdds.add(normalized);
args.push("--group-add", normalized);
for (const gid of extraGroupGids) {
if (!groupAdds.has(gid)) {
groupAdds.add(gid);
args.push("--group-add", gid);
}
}
for (const ulimit of dockerUlimits(inspect, options.requiredUlimits)) {
Expand Down Expand Up @@ -584,17 +600,32 @@ export function buildDockerGpuCloneRunArgs(

const entrypoint = stringArray(config.Entrypoint);
const replacementEntrypoint = String(options.containerEntrypoint ?? "").trim();
if (replacementEntrypoint) {
const groupBootstrapTarget = replacementEntrypoint || entrypoint[0] || "";
if (preserveJetsonGroups && groupBootstrapTarget !== OPENSHELL_SANDBOX_ENTRYPOINT) {
throw new Error("Jetson device-group bootstrap requires the OpenShell supervisor entrypoint.");
}
if (preserveJetsonGroups) {
args.push("--entrypoint", JETSON_DEVICE_GROUP_BOOTSTRAP);
} else if (replacementEntrypoint) {
args.push("--entrypoint", replacementEntrypoint);
} else if (entrypoint.length > 0) {
args.push("--entrypoint", entrypoint[0]);
}
const commandArgs = dockerContainerCommandArgs(
const targetCommandArgs = dockerContainerCommandArgs(
entrypoint,
stringArray(config.Cmd),
sandboxCommand,
options.containerCommand,
);
const commandArgs = preserveJetsonGroups
? [
"--device-group-gids",
extraGroupGids.join(","),
"--",
groupBootstrapTarget,
...targetCommandArgs,
]
: targetCommandArgs;
args.push(image, ...commandArgs);
return args;
}
Expand Down
Loading
Loading