Skip to content
E2E

E2E main #1

Sign in to view logs

E2E main

E2E main #1

Workflow file for this run

.github/workflows/e2e.yaml at c2a7292
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
name: E2E
run-name: "${{ inputs.checkout_sha != '' && format('E2E PR #{0} ({1})', inputs.pr_number, inputs.correlation_id) || inputs.correlation_id != '' && format('E2E {0} ({1})', github.ref_name, inputs.correlation_id) || format('E2E {0}', github.ref_name) }}"
on:
push:
branches: [main]
workflow_dispatch:
inputs:
targets:
description: "Optional comma-separated typed target IDs. On trusted main runs, empty jobs and targets select all supported registry E2E tests. PR revision runs use the trusted controller target matrix."
required: false
default: ""
type: string
jobs:
description: "Optional comma-separated E2E test IDs. Empty selectors choose the default suite. Set include_staging_brev_launchable for Launchable. Jetson dispatch and DGX Spark require their opt-in flags. PR revisions use the trusted controller matrix."
required: false
default: ""
type: string
include_staging_brev_launchable:
description: "Include Exact staging Brev Launchable in a full E2E run when jobs and targets are empty."
required: false
default: false
type: boolean
release_qualification_waived_jobs:
description: "Admin-only comma-separated release-required E2E job IDs to waive. Leave empty to require every release E2E job."
required: false
default: ""
type: string
release_qualification_waiver_reason:
description: "Admin-only reason for release qualification waived jobs. Required when release_qualification_waived_jobs is not empty."
required: false
default: ""
type: string
inference_mode:
description: "Inference adapter mode for compatible Vitest E2E jobs: mock, internal-nvidia, or public-nvidia."
required: false
default: "mock"
type: choice
options:
- mock
- internal-nvidia
- public-nvidia
allow_jetson_dispatch:
description: "Set true for a manual jetson-nvmap-gpu run. Main pushes dispatch it automatically. The operator-owned dispatch backend must be available, and JETSON_DISPATCH_URL must contain its verified HTTPS origin. Refer to test/e2e/docs/jetson-dispatch.md."
required: false
default: false
type: boolean
allow_dgx_spark_runner_queue:
description: "Before setting true for llama-cpp-dgx-spark-qualification, obtain repository administrator confirmation of an online DGX Spark runner in the authoritative NVIDIA/NemoClaw Settings -> Actions -> Runners inventory; queued jobs do not honor timeout-minutes before assignment."
required: false
default: false
type: boolean
pr_number:
description: Optional current PR number for manual exact-revision E2E.
required: false
type: string
default: ""
post_to_slack:
description: Post a selective-dispatch scorecard to the preview Slack route.
required: false
default: false
type: boolean
checkout_sha:
description: Optional lowercase 40-character latest PR commit SHA for manual E2E.
required: false
default: ""
type: string
checkout_repository:
description: Optional PR source repository for manual E2E.
required: false
default: ""
type: string
review_reason:
description: Optional review reason containing 10 to 500 printable characters.
required: false
default: ""
type: string
base_sha:
description: Optional lowercase PR base SHA for manual exact-revision E2E.
required: false
default: ""
type: string
workflow_sha:
description: Optional trusted main workflow SHA for manual E2E.
required: false
default: ""
type: string
correlation_id:
description: Run correlation ID (UUIDv4).
required: false
default: ""
type: string
permissions:
checks: read
contents: read
pull-requests: read
concurrency:
group: e2e-${{ github.ref }}-${{ inputs.checkout_sha != '' && format('pr-{0}', inputs.pr_number) || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '' && format('full-{0}', github.run_id)) || inputs.targets || 'supported' }}-${{ inputs.checkout_sha != '' && 'manual-pr' || inputs.jobs || 'all-jobs' }}
cancel-in-progress: ${{ inputs.checkout_sha != '' && !inputs.allow_jetson_dispatch }}
env:
NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }}
NEMOCLAW_E2E_CORRELATION_ID: ${{ inputs.correlation_id }}
NEMOCLAW_E2E_SHARD: default
jobs:
base-image-publication:
runs-on: ubuntu-latest
timeout-minutes: 55
outputs:
dcode_base_contract: ${{ steps.validate_dcode_base.outputs.contract }}
dcode_base_ref: ${{ steps.validate_dcode_base.outputs.base_ref }}
permissions:
actions: read
contents: read
steps:
- id: publication_mode
name: Classify base-image publication requirement
env:
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
EVENT_NAME: ${{ github.event_name }}
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
shell: bash
run: |
set -euo pipefail
case "${REPOSITORY}:${REF}:${EVENT_NAME}:${CHECKOUT_SHA:+controller}" in
NVIDIA/NemoClaw:refs/heads/main:push:|NVIDIA/NemoClaw:refs/heads/main:workflow_dispatch:)
required=1
;;
NVIDIA/NemoClaw:refs/heads/main:workflow_dispatch:controller)
required=1
;;
*)
echo "::error::base-image publication mode is not trusted" >&2
exit 1
;;
esac
printf 'required=%s\n' "${required}" >> "${GITHUB_OUTPUT}"
- name: Check out trusted E2E workflow
if: ${{ steps.publication_mode.outputs.required == '1' }}
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.sha }}
fetch-depth: 0
persist-credentials: false
- name: Set up Node for publication verification
if: ${{ steps.publication_mode.outputs.required == '1' }}
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
- id: publication
name: Verify applicable base-image publication
if: ${{ steps.publication_mode.outputs.required == '1' }}
env:
EXPECTED_SHA: ${{ github.sha }}
GITHUB_TOKEN: ${{ github.token }}
run: node --experimental-strip-types --no-warnings tools/e2e/base-image-publication.mts --wait-seconds 3000 --poll-seconds 30
- name: Download immutable Deep Agents Code base contract
if: ${{ steps.publication_mode.outputs.required == '1' }}
env:
GITHUB_TOKEN: ${{ github.token }}
PUBLICATION_HEAD_SHA: ${{ steps.publication.outputs.head_sha }}
PUBLICATION_RUN_ATTEMPT: ${{ steps.publication.outputs.run_attempt }}
PUBLICATION_RUN_ID: ${{ steps.publication.outputs.run_id }}
run: node --experimental-strip-types --no-warnings tools/e2e/exact-artifact-download.mts "${RUNNER_TEMP}/dcode-base-contract"
- id: validate_dcode_base
name: Validate immutable Deep Agents Code base
if: ${{ steps.publication_mode.outputs.required == '1' }}
env:
PUBLICATION_HEAD_SHA: ${{ steps.publication.outputs.head_sha }}
PUBLICATION_RUN_ATTEMPT: ${{ steps.publication.outputs.run_attempt }}
PUBLICATION_RUN_ID: ${{ steps.publication.outputs.run_id }}
run: node --experimental-strip-types --no-warnings tools/e2e/dcode-base-image-contract.mts "${RUNNER_TEMP}/dcode-base-contract/contract.json"
generate-matrix:
runs-on: ubuntu-latest
timeout-minutes: 10
outputs:
cli_artifact_provenance: ${{ steps.record_cli_artifact.outputs.provenance }}
e2e_credentials_allowed: ${{ steps.e2e_credentials.outputs.allowed }}
matrix: ${{ steps.matrix.outputs.matrix }}
test_matrix: ${{ steps.matrix.outputs.test_matrix }}
hermes_selected: ${{ steps.matrix.outputs.hermes_selected }}
explicit_only_jobs: ${{ steps.matrix.outputs.explicit_only_jobs }}
release_required_jobs: ${{ steps.matrix.outputs.release_required_jobs }}
release_qualification_waived_jobs: ${{ steps.matrix.outputs.release_qualification_waived_jobs }}
selected_jobs: ${{ steps.matrix.outputs.selected_jobs }}
selected_workflow_jobs: ${{ steps.matrix.outputs.selected_workflow_jobs }}
catalogue_standard_matrix: ${{ steps.matrix.outputs.catalogue_standard_matrix }}
catalogue_nvidia_api_matrix: ${{ steps.matrix.outputs.catalogue_nvidia_api_matrix }}
catalogue_nvidia_inference_matrix: ${{ steps.matrix.outputs.catalogue_nvidia_inference_matrix }}
catalogue_github_read_matrix: ${{ steps.matrix.outputs.catalogue_github_read_matrix }}
catalogue_brave_nvidia_inference_matrix: ${{ steps.matrix.outputs.catalogue_brave_nvidia_inference_matrix }}
runner_routing: ${{ steps.runner_routing.outputs.runner_routing }}
steps:
- id: controller_matrix
name: Build trusted controller target matrix
if: ${{ inputs.checkout_sha != '' }}
env:
JOBS: ${{ inputs.jobs }}
TARGETS: ${{ inputs.targets }}
shell: bash
run: |
set -euo pipefail
test_matrix='[]'
case "${JOBS}:${TARGETS}" in
:)
matrix='[{"id":"ubuntu-policy-custom-missing-presets-negative","runner":"ubuntu-latest"},{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest"},{"id":"ubuntu-repo-cloud-openclaw","runner":"ubuntu-latest"},{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest"}]'
test_matrix='[{"id":"onboard-managed-image-buildless-e2e","file":"test/onboard-managed-image-buildless-e2e.test.ts","project":"integration"},{"id":"vllm-docker-storage","file":"test/vllm-docker-storage.test.ts","project":"integration"}]'
;;
inference-routing: | managed-image-protected-runtime: | native-runtime-qualification-producer: | :jetson-nvmap-gpu)
matrix='[]'
;;
:ubuntu-repo-cloud-langchain-deepagents-code)
matrix='[{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest","label":"ubuntu-repo-cloud-langchain-deepagents-code"}]'
;;
:ubuntu-repo-docker-post-reboot-recovery)
matrix='[{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest","label":"ubuntu-repo-docker-post-reboot-recovery"}]'
;;
:ubuntu-repo-cloud-langchain-deepagents-code,ubuntu-repo-docker-post-reboot-recovery)
matrix='[{"id":"ubuntu-repo-cloud-langchain-deepagents-code","runner":"ubuntu-latest","label":"ubuntu-repo-cloud-langchain-deepagents-code"},{"id":"ubuntu-repo-docker-post-reboot-recovery","runner":"ubuntu-latest","label":"ubuntu-repo-docker-post-reboot-recovery"}]'
;;
*)
echo "::error::PR E2E target is not approved by the trusted controller" >&2
exit 1
;;
esac
printf 'matrix=%s\n' "${matrix}" >> "${GITHUB_OUTPUT}"
printf 'test_matrix=%s\n' "${test_matrix}" >> "${GITHUB_OUTPUT}"
- id: runner_routing
name: Build trusted larger-runner routing
env:
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
LARGER_RUNNER_LABEL: ${{ vars.E2E_LARGER_RUNNER_LABEL }}
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
shell: bash
run: |
set -euo pipefail
larger_runner="ubuntu-latest"
if [[ "${REPOSITORY}" == "NVIDIA/NemoClaw" && "${REF}" == "refs/heads/main" && -z "${CHECKOUT_SHA}" && -n "${LARGER_RUNNER_LABEL}" ]]; then
if [[ ! "${LARGER_RUNNER_LABEL}" =~ ^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$ ]]; then
echo "::error::E2E_LARGER_RUNNER_LABEL must be a 1-64 character workflow label using letters, digits, dots, underscores, or hyphens" >&2
exit 1
fi
larger_runner="${LARGER_RUNNER_LABEL}"
fi
runner_routing="$(jq -cn --arg standard "ubuntu-latest" --arg larger "${larger_runner}" '{"channels-stop-start-hermes":$larger,"common-egress-agent":$larger,"hermes-discord":$larger,"hermes-e2e":$larger,"hermes-inference-switch":$larger,"hermes-shields-config":$larger,"mcp-bridge-deepagents":$larger,"mcp-bridge-hermes":$larger,"mcp-bridge-openclaw":$standard,"rebuild-hermes":$larger,"rebuild-hermes-stale-base":$larger,"security-posture-hermes":$larger}')"
printf 'runner_routing=%s\n' "${runner_routing}" >> "${GITHUB_OUTPUT}"
- name: Authenticate manual PR dispatch
if: ${{ inputs.checkout_sha != '' }}
env:
ACTOR: ${{ github.actor }}
ALLOW_JETSON_DISPATCH: ${{ inputs.allow_jetson_dispatch && 'true' || 'false' }}
BASE_SHA: ${{ inputs.base_sha }}
CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }}
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
GITHUB_TOKEN: ${{ github.token }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }}
INCLUDE_LAUNCHABLE: ${{ inputs.include_staging_brev_launchable }}
JOBS: ${{ inputs.jobs }}
PR_NUMBER: ${{ inputs.pr_number }}
REVIEW_REASON: ${{ inputs.review_reason }}
RUN_ATTEMPT: ${{ github.run_attempt }}
TARGETS: ${{ inputs.targets }}
TRIGGERING_ACTOR: ${{ github.triggering_actor }}
WORKFLOW_EVENT: ${{ github.event_name }}
WORKFLOW_REF: ${{ github.ref }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
read_collaborator_permission() {
local maintainer="$1"
local attempt curl_exit failure http_status permission_file
permission_file="$(mktemp "${RUNNER_TEMP:-/tmp}/nemoclaw-collaborator-permission.XXXXXX")"
for attempt in 1 2 3; do
: >"$permission_file"
if http_status="$(curl --silent --proto '=https' --connect-timeout 10 --max-time 30 \
--output "$permission_file" --write-out "%{http_code}" \
--header "Authorization: Bearer ${GITHUB_TOKEN}" \
--header "Accept: application/vnd.github+json" \
--header "X-GitHub-Api-Version: 2022-11-28" \
"https://api.github.com/repos/${GITHUB_REPOSITORY}/collaborators/${maintainer}/permission" \
2>/dev/null)"; then
if [[ "$http_status" =~ ^2[0-9]{2}$ ]]; then
if jq -e 'type == "object" and (.user.login | type == "string") and (.role_name | type == "string")' "$permission_file" >/dev/null 2>&1; then
if (( attempt > 1 )); then
echo "::notice::Collaborator permission read passed after retry on attempt ${attempt}/3" >&2
fi
cat "$permission_file"
rm -f "$permission_file"
return 0
fi
echo "::error::Collaborator permission read attempt ${attempt}/3 failed: malformed response" >&2
rm -f "$permission_file"
return 1
fi
if [[ "$http_status" =~ ^[0-9]{3}$ ]]; then
failure="HTTP ${http_status}"
case "$http_status" in
408 | 429 | 5??) ;;
*) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: ${failure}" >&2; rm -f "$permission_file"; return 1 ;;
esac
else
echo "::error::Collaborator permission read attempt ${attempt}/3 failed: invalid HTTP status" >&2
rm -f "$permission_file"
return 1
fi
else
curl_exit=$?
case "$curl_exit" in
5 | 6 | 7 | 16 | 18 | 28 | 35 | 52 | 55 | 56 | 92 | 95 | 96) failure="transport" ;;
*) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: curl exit ${curl_exit}" >&2; rm -f "$permission_file"; return 1 ;;
esac
fi
if (( attempt == 3 )); then
echo "::error::Collaborator permission read exhausted after attempt ${attempt}/3: ${failure}" >&2
rm -f "$permission_file"
return 1
fi
echo "::warning::Collaborator permission read attempt ${attempt}/3 failed: ${failure}; retrying" >&2
sleep "$attempt"
done
}
require_maintainer() {
local maintainer="$1"
[[ "$maintainer" =~ ^[A-Za-z0-9-]{1,39}$ && "$maintainer" != -* && "$maintainer" != *- ]] || {
echo "::error::Manual PR E2E actor is invalid" >&2
exit 1
}
local permission_json
permission_json="$(read_collaborator_permission "$maintainer")"
if [[ "$(jq -r '.user.login // ""' <<< "$permission_json" | tr '[:upper:]' '[:lower:]')" != "$(tr '[:upper:]' '[:lower:]' <<< "$maintainer")" ]]; then
echo "::error::Manual PR E2E permission response did not match the actor" >&2
exit 1
fi
case "$(jq -r '.role_name // ""' <<< "$permission_json")" in
maintain | admin) ;;
*) echo "::error::Manual PR E2E requires a repository maintainer or administrator" >&2; exit 1 ;;
esac
}
[[ "$WORKFLOW_EVENT" == "workflow_dispatch" && "$WORKFLOW_REF" == "refs/heads/main" ]] || {
echo "::error::Manual PR E2E must be dispatched from main" >&2
exit 1
}
[[ "$RUN_ATTEMPT" == "1" ]] || { echo "::error::Manual PR E2E cannot be rerun" >&2; exit 1; }
case "${JOBS}:${TARGETS}:${INCLUDE_LAUNCHABLE}:${ALLOW_JETSON_DISPATCH}" in
::false:false | inference-routing::false:false | managed-image-protected-runtime::false:false | native-runtime-qualification-producer::false:false | :jetson-nvmap-gpu:false:true) ;;
*)
echo "::error::Manual PR E2E accepts only empty selectors, inference-routing, managed-image-protected-runtime, native-runtime-qualification-producer, or jetson-nvmap-gpu with its dispatch flag" >&2
exit 1
;;
esac
[[ "$PR_NUMBER" =~ ^[1-9][0-9]*$ ]] || { echo "::error::pr_number must be a positive integer" >&2; exit 1; }
[[ "$CHECKOUT_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] || { echo "::error::checkout_repository must be an owner/repository name" >&2; exit 1; }
[[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::checkout_sha must be a lowercase 40-character SHA" >&2; exit 1; }
[[ "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || { echo "::error::base_sha must be a lowercase 40-character SHA" >&2; exit 1; }
[[ "$REVIEW_REASON" =~ ^[[:print:]]+$ ]] &&
(( ${#REVIEW_REASON} >= 10 && ${#REVIEW_REASON} <= 500 )) ||
{ echo "::error::review_reason must contain 10 to 500 printable characters" >&2; exit 1; }
[[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA" ]] || { echo "::error::workflow_sha must match the trusted main workflow SHA" >&2; exit 1; }
require_maintainer "$ACTOR"
if [[ "$(printf '%s' "$TRIGGERING_ACTOR" | tr '[:upper:]' '[:lower:]')" != "$(printf '%s' "$ACTOR" | tr '[:upper:]' '[:lower:]')" ]]; then
require_maintainer "$TRIGGERING_ACTOR"
fi
pull_json="$(curl --fail --silent --show-error --proto '=https' \
--header "Authorization: Bearer ${GITHUB_TOKEN}" \
--header "Accept: application/vnd.github+json" \
--header "X-GitHub-Api-Version: 2022-11-28" \
"https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")"
[[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must be open" >&2; exit 1; }
[[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository must match the PR source repository" >&2; exit 1; }
[[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha must match the latest PR commit SHA" >&2; exit 1; }
[[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha must match the PR base SHA" >&2; exit 1; }
- name: Authorize release qualification waiver
if: ${{ github.event_name == 'workflow_dispatch' && (inputs.release_qualification_waived_jobs != '' || inputs.release_qualification_waiver_reason != '') }}
env:
ACTOR: ${{ github.actor }}
ALLOW_DGX_SPARK_RUNNER_QUEUE: ${{ inputs.allow_dgx_spark_runner_queue && 'true' || 'false' }}
ALLOW_JETSON_DISPATCH: ${{ inputs.allow_jetson_dispatch && 'true' || 'false' }}
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
GITHUB_TOKEN: ${{ github.token }}
INCLUDE_LAUNCHABLE: ${{ inputs.include_staging_brev_launchable && 'true' || 'false' }}
JOBS: ${{ inputs.jobs }}
TARGETS: ${{ inputs.targets }}
TRIGGERING_ACTOR: ${{ github.triggering_actor }}
WAIVED_JOBS: ${{ inputs.release_qualification_waived_jobs }}
WAIVER_REASON: ${{ inputs.release_qualification_waiver_reason }}
WORKFLOW_REF: ${{ github.ref }}
shell: bash
run: |
set -euo pipefail
read_collaborator_permission() {
local administrator="$1"
local attempt curl_exit failure http_status permission_file
permission_file="$(mktemp "${RUNNER_TEMP:-/tmp}/nemoclaw-collaborator-permission.XXXXXX")"
for attempt in 1 2 3; do
: >"$permission_file"
if http_status="$(curl --silent --proto '=https' --connect-timeout 10 --max-time 30 \
--output "$permission_file" --write-out "%{http_code}" \
--header "Authorization: Bearer ${GITHUB_TOKEN}" \
--header "Accept: application/vnd.github+json" \
--header "X-GitHub-Api-Version: 2022-11-28" \
"https://api.github.com/repos/${GITHUB_REPOSITORY}/collaborators/${administrator}/permission" \
2>/dev/null)"; then
if [[ "$http_status" =~ ^2[0-9]{2}$ ]]; then
if jq -e 'type == "object" and (.user.login | type == "string") and (.role_name | type == "string")' "$permission_file" >/dev/null 2>&1; then
if (( attempt > 1 )); then
echo "::notice::Collaborator permission read passed after retry on attempt ${attempt}/3" >&2
fi
cat "$permission_file"
rm -f "$permission_file"
return 0
fi
echo "::error::Collaborator permission read attempt ${attempt}/3 failed: malformed response" >&2
rm -f "$permission_file"
return 1
fi
if [[ "$http_status" =~ ^[0-9]{3}$ ]]; then
failure="HTTP ${http_status}"
case "$http_status" in
408 | 429 | 5??) ;;
*) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: ${failure}" >&2; rm -f "$permission_file"; return 1 ;;
esac
else
echo "::error::Collaborator permission read attempt ${attempt}/3 failed: invalid HTTP status" >&2
rm -f "$permission_file"
return 1
fi
else
curl_exit=$?
case "$curl_exit" in
5 | 6 | 7 | 16 | 18 | 28 | 35 | 52 | 55 | 56 | 92 | 95 | 96) failure="transport" ;;
*) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: curl exit ${curl_exit}" >&2; rm -f "$permission_file"; return 1 ;;
esac
fi
if (( attempt == 3 )); then
echo "::error::Collaborator permission read exhausted after attempt ${attempt}/3: ${failure}" >&2
rm -f "$permission_file"
return 1
fi
echo "::warning::Collaborator permission read attempt ${attempt}/3 failed: ${failure}; retrying" >&2
sleep "$attempt"
done
}
require_admin() {
local administrator="$1"
if [[ ! "$administrator" =~ ^[A-Za-z0-9-]{1,39}$ || "$administrator" == -* || "$administrator" == *- ]]; then
echo "::error::Release qualification waiver actor is invalid" >&2
exit 1
fi
local permission_json
permission_json="$(read_collaborator_permission "$administrator")"
if [[ "$(jq -r '.user.login // ""' <<< "$permission_json" | tr '[:upper:]' '[:lower:]')" != "$(tr '[:upper:]' '[:lower:]' <<< "$administrator")" ]]; then
echo "::error::Release qualification waiver permission response did not match the actor" >&2
exit 1
fi
[[ "$(jq -r '.role_name // ""' <<< "$permission_json")" == "admin" ]] || {
echo "::error::Release qualification waiver requires a repository administrator" >&2
exit 1
}
}
[[ "$WORKFLOW_REF" == "refs/heads/main" ]] || {
echo "::error::Release qualification waiver must be dispatched from main" >&2
exit 1
}
[[ -z "$CHECKOUT_SHA" && -z "$JOBS" && -z "$TARGETS" ]] || {
echo "::error::Release qualification waiver requires an exact main release run with empty selectors" >&2
exit 1
}
[[ "$INCLUDE_LAUNCHABLE" == "true" && "$ALLOW_JETSON_DISPATCH" == "false" && "$ALLOW_DGX_SPARK_RUNNER_QUEUE" == "false" ]] || {
echo "::error::Release qualification waiver conflicts with another release-run override" >&2
exit 1
}
waived_jobs_pattern='^[a-z0-9]+(-[a-z0-9]+)*(,[a-z0-9]+(-[a-z0-9]+)*)*$'
[[ "$WAIVED_JOBS" =~ $waived_jobs_pattern ]] || {
echo "::error::release_qualification_waived_jobs must contain comma-separated E2E job IDs" >&2
exit 1
}
waiver_reason_pattern="^[A-Za-z0-9][A-Za-z0-9 .,:;/_()'-]*$"
[[ ${#WAIVER_REASON} -ge 10 && ${#WAIVER_REASON} -le 500 && "$WAIVER_REASON" =~ $waiver_reason_pattern ]] || {
echo "::error::release_qualification_waiver_reason must start with a letter or number and contain 10 to 500 allowed ASCII characters" >&2
exit 1
}
require_admin "$ACTOR"
if [[ "$(printf '%s' "$TRIGGERING_ACTOR" | tr '[:upper:]' '[:lower:]')" != "$(printf '%s' "$ACTOR" | tr '[:upper:]' '[:lower:]')" ]]; then
require_admin "$TRIGGERING_ACTOR"
fi
- name: Record trusted E2E dispatch receipt
if: ${{ github.event_name == 'workflow_dispatch' }}
env:
ACTOR: ${{ github.actor }}
ALLOW_DGX_SPARK_RUNNER_QUEUE: ${{ inputs.allow_dgx_spark_runner_queue && 'true' || 'false' }}
ALLOW_JETSON_DISPATCH: ${{ inputs.allow_jetson_dispatch && 'true' || 'false' }}
ALLOW_JETSON_RUNNER_QUEUE: "false"
BASE_SHA: ${{ inputs.checkout_sha != '' && inputs.base_sha || github.sha }}
CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository || github.repository }}
CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }}
DISPATCH_JOBS: ${{ inputs.jobs }}
DISPATCH_RECEIPT_DIR: ${{ runner.temp }}/nemoclaw-e2e-dispatch
DISPATCH_TARGETS: ${{ inputs.targets }}
EVENT_NAME: ${{ github.event_name }}
INCLUDE_STAGING_BREV_LAUNCHABLE: ${{ inputs.include_staging_brev_launchable && 'true' || 'false' }}
PR_NUMBER: ${{ inputs.checkout_sha != '' && inputs.pr_number || '' }}
REPOSITORY: ${{ github.repository }}
RUN_ATTEMPT: ${{ github.run_attempt }}
RUN_ID: ${{ github.run_id }}
RELEASE_QUALIFICATION_WAIVED_JOBS: ${{ inputs.release_qualification_waived_jobs }}
RELEASE_QUALIFICATION_WAIVER_REASON: ${{ inputs.release_qualification_waiver_reason }}
TRIGGERING_ACTOR: ${{ github.triggering_actor }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
install -d -m 0700 "$DISPATCH_RECEIPT_DIR"
jq -n \
--arg actor "$ACTOR" \
--arg baseSha "$BASE_SHA" \
--arg candidateRepository "$CANDIDATE_REPOSITORY" \
--arg candidateSha "$CANDIDATE_SHA" \
--arg eventName "$EVENT_NAME" \
--arg jobs "$DISPATCH_JOBS" \
--arg prNumber "$PR_NUMBER" \
--arg repository "$REPOSITORY" \
--arg releaseQualificationWaivedJobs "$RELEASE_QUALIFICATION_WAIVED_JOBS" \
--arg releaseQualificationWaiverReason "$RELEASE_QUALIFICATION_WAIVER_REASON" \
--arg targets "$DISPATCH_TARGETS" \
--arg triggeringActor "$TRIGGERING_ACTOR" \
--arg workflowRunId "$RUN_ID" \
--arg workflowSha "$WORKFLOW_SHA" \
--argjson allowDgxSparkRunnerQueue "$ALLOW_DGX_SPARK_RUNNER_QUEUE" \
--argjson allowJetsonDispatch "$ALLOW_JETSON_DISPATCH" \
--argjson allowJetsonRunnerQueue "$ALLOW_JETSON_RUNNER_QUEUE" \
--argjson includeStagingBrevLaunchable "$INCLUDE_STAGING_BREV_LAUNCHABLE" \
--argjson workflowRunAttempt "$RUN_ATTEMPT" \
'{
kind: "nemoclaw-e2e-dispatch-v2",
actor: $actor,
repository: $repository,
prNumber: (if $prNumber == "" then null else ($prNumber | tonumber) end),
candidateRepository: $candidateRepository,
candidateSha: $candidateSha,
baseSha: $baseSha,
workflowSha: $workflowSha,
workflowRunId: $workflowRunId,
workflowRunAttempt: $workflowRunAttempt,
eventName: $eventName,
jobs: $jobs,
targets: $targets,
allowDgxSparkRunnerQueue: $allowDgxSparkRunnerQueue,
allowJetsonDispatch: $allowJetsonDispatch,
allowJetsonRunnerQueue: $allowJetsonRunnerQueue,
includeStagingBrevLaunchable: $includeStagingBrevLaunchable,
releaseQualificationWaivedJobs: (if $releaseQualificationWaivedJobs == "" then [] else ($releaseQualificationWaivedJobs | split(",")) end),
releaseQualificationWaiverReason: (if $releaseQualificationWaiverReason == "" then null else $releaseQualificationWaiverReason end),
triggeringActor: $triggeringActor,
emptySelectors: ($jobs == "" and $targets == "")
}' >"$DISPATCH_RECEIPT_DIR/dispatch.json"
- name: Upload trusted E2E dispatch receipt
if: ${{ github.event_name == 'workflow_dispatch' }}
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-dispatch-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/nemoclaw-e2e-dispatch/dispatch.json
- name: Authorize Launchable E2E maintainer dispatch
if: ${{ github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }}
env:
ACTOR: ${{ github.actor }}
GITHUB_TOKEN: ${{ github.token }}
TRIGGERING_ACTOR: ${{ github.triggering_actor }}
shell: bash
run: |
set -euo pipefail
read_collaborator_permission() {
local maintainer="$1"
local attempt curl_exit failure http_status permission_file
permission_file="$(mktemp "${RUNNER_TEMP:-/tmp}/nemoclaw-collaborator-permission.XXXXXX")"
for attempt in 1 2 3; do
: >"$permission_file"
if http_status="$(curl --silent --proto '=https' --connect-timeout 10 --max-time 30 \
--output "$permission_file" --write-out "%{http_code}" \
--header "Authorization: Bearer ${GITHUB_TOKEN}" \
--header "Accept: application/vnd.github+json" \
--header "X-GitHub-Api-Version: 2022-11-28" \
"https://api.github.com/repos/${GITHUB_REPOSITORY}/collaborators/${maintainer}/permission" \
2>/dev/null)"; then
if [[ "$http_status" =~ ^2[0-9]{2}$ ]]; then
if jq -e 'type == "object" and (.user.login | type == "string") and (.role_name | type == "string")' "$permission_file" >/dev/null 2>&1; then
if (( attempt > 1 )); then
echo "::notice::Collaborator permission read passed after retry on attempt ${attempt}/3" >&2
fi
cat "$permission_file"
rm -f "$permission_file"
return 0
fi
echo "::error::Collaborator permission read attempt ${attempt}/3 failed: malformed response" >&2
rm -f "$permission_file"
return 1
fi
if [[ "$http_status" =~ ^[0-9]{3}$ ]]; then
failure="HTTP ${http_status}"
case "$http_status" in
408 | 429 | 5??) ;;
*) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: ${failure}" >&2; rm -f "$permission_file"; return 1 ;;
esac
else
echo "::error::Collaborator permission read attempt ${attempt}/3 failed: invalid HTTP status" >&2
rm -f "$permission_file"
return 1
fi
else
curl_exit=$?
case "$curl_exit" in
5 | 6 | 7 | 16 | 18 | 28 | 35 | 52 | 55 | 56 | 92 | 95 | 96) failure="transport" ;;
*) echo "::error::Collaborator permission read attempt ${attempt}/3 failed: curl exit ${curl_exit}" >&2; rm -f "$permission_file"; return 1 ;;
esac
fi
if (( attempt == 3 )); then
echo "::error::Collaborator permission read exhausted after attempt ${attempt}/3: ${failure}" >&2
rm -f "$permission_file"
return 1
fi
echo "::warning::Collaborator permission read attempt ${attempt}/3 failed: ${failure}; retrying" >&2
sleep "$attempt"
done
}
require_maintainer() {
local maintainer="$1"
if [[ ! "$maintainer" =~ ^[A-Za-z0-9-]{1,39}$ || "$maintainer" == -* || "$maintainer" == *- ]]; then
echo "::error::Launchable E2E actor is invalid" >&2
exit 1
fi
local permission_json
permission_json="$(read_collaborator_permission "$maintainer")"
if [[ "$(jq -r '.user.login // ""' <<< "$permission_json" | tr '[:upper:]' '[:lower:]')" != "$(tr '[:upper:]' '[:lower:]' <<< "$maintainer")" ]]; then
echo "::error::Launchable E2E permission response did not match the actor" >&2
exit 1
fi
case "$(jq -r '.role_name // ""' <<< "$permission_json")" in
maintain | admin) ;;
*)
echo "::error::Launchable E2E requires a repository maintainer or administrator" >&2
exit 1
;;
esac
}
require_maintainer "$ACTOR"
if [[ "$(printf '%s' "$TRIGGERING_ACTOR" | tr '[:upper:]' '[:lower:]')" != "$(printf '%s' "$ACTOR" | tr '[:upper:]' '[:lower:]')" ]]; then
require_maintainer "$TRIGGERING_ACTOR"
fi
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
if: ${{ inputs.checkout_sha == '' || inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '' }}
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- name: Validate manual PR checkout
if: ${{ inputs.checkout_sha != '' && (inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '') }}
env:
BASE_SHA: ${{ inputs.base_sha }}
CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }}
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
GITHUB_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ inputs.pr_number }}
shell: bash
run: |
set -euo pipefail
[[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || { echo "::error::checked-out commit does not match checkout_sha" >&2; exit 1; }
pull_json="$(curl --fail --silent --show-error --proto '=https' \
--header "Authorization: Bearer ${GITHUB_TOKEN}" \
--header "Accept: application/vnd.github+json" \
--header "X-GitHub-Api-Version: 2022-11-28" \
"https://api.github.com/repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")"
[[ "$(jq -r '.state' <<< "$pull_json")" == "open" ]] || { echo "::error::pull request must still be open" >&2; exit 1; }
[[ "$(jq -r '.head.repo.full_name // ""' <<< "$pull_json")" == "$CHECKOUT_REPOSITORY" ]] || { echo "::error::checkout_repository changed before execution" >&2; exit 1; }
[[ "$(jq -r '.head.sha' <<< "$pull_json")" == "$CHECKOUT_SHA" ]] || { echo "::error::checkout_sha changed before execution" >&2; exit 1; }
[[ "$(jq -r '.base.sha' <<< "$pull_json")" == "$BASE_SHA" ]] || { echo "::error::base_sha changed before execution" >&2; exit 1; }
- id: e2e_credentials
name: Authorize E2E credentials
if: ${{ inputs.checkout_sha != '' && (inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '') }}
env:
CHECKOUT_REPOSITORY: ${{ inputs.checkout_repository }}
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }}
REF: ${{ github.ref }}
WORKFLOW_REPOSITORY: ${{ github.repository }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
credentials_allowed=false
if [[ "$WORKFLOW_REPOSITORY" == "NVIDIA/NemoClaw" &&
"$CHECKOUT_REPOSITORY" == "$WORKFLOW_REPOSITORY" &&
"$EVENT_NAME" == "workflow_dispatch" &&
"$REF" == "refs/heads/main" &&
"$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ &&
"$WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ &&
"$EXPECTED_WORKFLOW_SHA" == "$WORKFLOW_SHA" &&
"$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]]; then
credentials_allowed=true
fi
printf 'allowed=%s\n' "$credentials_allowed" >> "$GITHUB_OUTPUT"
- name: Prepare E2E workspace
if: ${{ inputs.checkout_sha == '' || inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '' }}
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
- id: package_cli_artifact
name: Package exact-commit CLI
if: ${{ inputs.checkout_sha == '' || inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '' }}
env:
CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository || github.repository }}
CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }}
RUN_ATTEMPT: ${{ github.run_attempt }}
RUN_ID: ${{ github.run_id }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
[[ "$CANDIDATE_REPOSITORY" =~ ^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$ ]] ||
{ echo "::error::candidate repository is invalid"; exit 1; }
[[ "$CANDIDATE_SHA" =~ ^[a-f0-9]{40}$ ]] ||
{ echo "::error::candidate SHA must be a lowercase 40-character SHA"; exit 1; }
[[ "$WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ ]] ||
{ echo "::error::workflow SHA must be a lowercase 40-character SHA"; exit 1; }
[[ "$(git rev-parse --verify HEAD)" == "$CANDIDATE_SHA" ]] ||
{ echo "::error::checked-out commit does not match the artifact candidate SHA"; exit 1; }
[[ "$RUN_ATTEMPT" =~ ^[1-9][0-9]*$ && "$RUN_ID" =~ ^[1-9][0-9]*$ ]] ||
{ echo "::error::workflow run identity is invalid"; exit 1; }
for required_file in dist/nemoclaw.js dist/build-identity.json; do
[[ -f "$required_file" && ! -L "$required_file" && -s "$required_file" ]] ||
{ echo "::error::candidate CLI build output is missing or is not a nonempty regular file: $required_file"; exit 1; }
done
for boundary in \
openshell-policy-boundary.cjs \
sandbox-name.cjs \
snapshot-sanitizer-boundary.cjs; do
boundary_path="nemoclaw/dist/shared/$boundary"
[[ -f "$boundary_path" && ! -L "$boundary_path" && -s "$boundary_path" ]] ||
{ echo "::error::candidate CLI build shared module is missing or is not a nonempty regular file: $boundary"; exit 1; }
done
jq -e --arg candidateSha "$CANDIDATE_SHA" '
type == "object" and
(keys | sort) == ["nemoclawVersion", "sourceRevision"] and
(.nemoclawVersion | strings | length > 0) and
.sourceRevision == $candidateSha
' dist/build-identity.json >/dev/null ||
{ echo "::error::candidate CLI build identity does not match the candidate commit SHA"; exit 1; }
artifact_dir="${RUNNER_TEMP}/nemoclaw-cli-artifact"
install -d -m 0700 "$artifact_dir"
payload="$artifact_dir/nemoclaw-cli.tar"
manifest="$artifact_dir/manifest.json"
tar \
--sort=name \
--mtime=@0 \
--owner=0 \
--group=0 \
--numeric-owner \
-cf "$payload" \
dist \
nemoclaw/dist/shared
payload_sha256="$(sha256sum "$payload" | awk '{print $1}')"
source_tree="$(git rev-parse 'HEAD^{tree}')"
lockfile_sha256="$(sha256sum package-lock.json | awk '{print $1}')"
node_version="$(node --version)"
npm_version="$(npm --version)"
artifact_name="nemoclaw-cli-${CANDIDATE_SHA}-${payload_sha256}"
jq -n \
--arg artifactName "$artifact_name" \
--arg buildCommand "npm run build:cli" \
--arg candidateRepository "$CANDIDATE_REPOSITORY" \
--arg candidateSha "$CANDIDATE_SHA" \
--arg lockfileSha256 "$lockfile_sha256" \
--arg nodeVersion "$node_version" \
--arg npmVersion "$npm_version" \
--arg payloadFile "nemoclaw-cli.tar" \
--arg payloadSha256 "$payload_sha256" \
--arg runAttempt "$RUN_ATTEMPT" \
--arg runId "$RUN_ID" \
--arg runnerArch "$RUNNER_ARCH" \
--arg runnerOs "$RUNNER_OS" \
--arg sourceTree "$source_tree" \
--arg workflowSha "$WORKFLOW_SHA" \
'{
kind: "nemoclaw-e2e-cli-artifact-v1",
artifactName: $artifactName,
candidate: {
repository: $candidateRepository,
sha: $candidateSha,
sourceTree: $sourceTree,
lockfileSha256: $lockfileSha256
},
workflow: {
sha: $workflowSha,
runId: $runId,
runAttempt: $runAttempt
},
toolchain: {
node: $nodeVersion,
npm: $npmVersion,
runnerOs: $runnerOs,
runnerArch: $runnerArch
},
build: {
command: $buildCommand,
sourceRevision: $candidateSha
},
payload: {
file: $payloadFile,
sha256: $payloadSha256
}
}' >"$manifest"
chmod 0600 "$manifest" "$payload"
printf 'artifact_name=%s\n' "$artifact_name" >>"$GITHUB_OUTPUT"
printf 'candidate_sha=%s\n' "$CANDIDATE_SHA" >>"$GITHUB_OUTPUT"
printf 'payload_sha256=%s\n' "$payload_sha256" >>"$GITHUB_OUTPUT"
- id: upload_cli_artifact
name: Publish content-addressed CLI artifact
if: ${{ inputs.checkout_sha == '' || inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '' }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ${{ steps.package_cli_artifact.outputs.artifact_name }}
path: ${{ runner.temp }}/nemoclaw-cli-artifact/
if-no-files-found: error
retention-days: 3
compression-level: 0
- id: record_cli_artifact
name: Record CLI artifact provenance
if: ${{ inputs.checkout_sha == '' || inputs.jobs != 'native-runtime-qualification-producer' || inputs.targets != '' }}
env:
ARTIFACT_DIGEST: ${{ steps.upload_cli_artifact.outputs.artifact-digest }}
ARTIFACT_ID: ${{ steps.upload_cli_artifact.outputs.artifact-id }}
ARTIFACT_NAME: ${{ steps.package_cli_artifact.outputs.artifact_name }}
CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository || github.repository }}
CANDIDATE_SHA: ${{ steps.package_cli_artifact.outputs.candidate_sha }}
PAYLOAD_SHA256: ${{ steps.package_cli_artifact.outputs.payload_sha256 }}
RUN_ATTEMPT: ${{ github.run_attempt }}
RUN_ID: ${{ github.run_id }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
[[ "$ARTIFACT_ID" =~ ^[1-9][0-9]*$ ]] ||
{ echo "::error::artifact upload did not return an immutable artifact ID"; exit 1; }
[[ "$ARTIFACT_DIGEST" =~ ^[a-f0-9]{64}$ ]] ||
{ echo "::error::artifact upload did not return a SHA-256 digest"; exit 1; }
provenance="$(jq -cn \
--arg artifactDigest "$ARTIFACT_DIGEST" \
--arg artifactId "$ARTIFACT_ID" \
--arg artifactName "$ARTIFACT_NAME" \
--arg candidateRepository "$CANDIDATE_REPOSITORY" \
--arg candidateSha "$CANDIDATE_SHA" \
--arg payloadSha256 "$PAYLOAD_SHA256" \
--arg runAttempt "$RUN_ATTEMPT" \
--arg runId "$RUN_ID" \
--arg workflowSha "$WORKFLOW_SHA" \
'{
kind: "nemoclaw-e2e-cli-provenance-v1",
artifactDigest: $artifactDigest,
artifactId: $artifactId,
artifactName: $artifactName,
candidateRepository: $candidateRepository,
candidateSha: $candidateSha,
payloadSha256: $payloadSha256,
workflowSha: $workflowSha,
runId: $runId,
runAttempt: $runAttempt
}')"
printf 'provenance=%s\n' "$provenance" >>"$GITHUB_OUTPUT"
{
echo "## Exact-commit CLI artifact"
echo
echo "- Candidate: \`${CANDIDATE_SHA}\`"
echo "- Artifact: \`${ARTIFACT_NAME}\` (ID \`${ARTIFACT_ID}\`)"
echo "- GitHub archive digest: \`${ARTIFACT_DIGEST}\`"
echo "- Payload digest: \`${PAYLOAD_SHA256}\`"
} >>"$GITHUB_STEP_SUMMARY"
- id: matrix
name: Generate E2E target matrix
env:
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
CONTROLLER_MATRIX: ${{ steps.controller_matrix.outputs.matrix }}
CONTROLLER_TEST_MATRIX: ${{ steps.controller_matrix.outputs.test_matrix }}
INFERENCE_MODE: ${{ inputs.inference_mode || 'mock' }}
JOBS: ${{ inputs.jobs }}
TARGETS: ${{ inputs.targets }}
EVENT_NAME: ${{ github.event_name }}
BEFORE_SHA: ${{ github.event.before }}
CANDIDATE_SHA: ${{ github.sha }}
RELEASE_QUALIFICATION_WAIVED_JOBS: ${{ inputs.release_qualification_waived_jobs }}
run: |
set -euo pipefail
if [[ "${JOBS:-}" == "native-runtime-qualification-producer" && -z "${TARGETS:-}" ]]; then
{
echo 'matrix=[]'
echo 'test_matrix=[]'
echo 'catalogue_standard_matrix=[]'
echo 'catalogue_nvidia_api_matrix=[]'
echo 'catalogue_nvidia_inference_matrix=[]'
echo 'catalogue_github_read_matrix=[]'
echo 'catalogue_brave_nvidia_inference_matrix=[]'
echo 'selected_jobs=["native-runtime-qualification-producer"]'
echo 'selected_workflow_jobs=["native-runtime-qualification-producer"]'
echo 'hermes_selected=false'
echo 'explicit_only_jobs=native-runtime-qualification-producer'
echo 'release_qualification_waived_jobs=[]'
echo 'release_required_jobs=[]'
} >>"$GITHUB_OUTPUT"
exit 0
fi
if [ "${EVENT_NAME:-}" = "push" ]; then
[[ "${BEFORE_SHA}" =~ ^[a-f0-9]{40}$ && "${CANDIDATE_SHA}" =~ ^[a-f0-9]{40}$ ]] || {
echo "::error::Main push E2E requires exact before and candidate commit SHAs" >&2
exit 1
}
CHANGED_FILES="$(git diff --name-only --diff-filter=ACMRD "${BEFORE_SHA}" "${CANDIDATE_SHA}")"
export CHANGED_FILES
fi
npx tsx tools/e2e/workflow-plan.mts --ci-output
if [ -n "${CHECKOUT_SHA}" ]; then
matrix="$(sed -n 's/^matrix=//p' "${GITHUB_OUTPUT}")"
expected_controller_matrix="$(jq -c 'map({id, runner}) | sort_by(.id)' <<< "${CONTROLLER_MATRIX}")"
actual_controller_matrix="$(jq -c 'map({id, runner}) | sort_by(.id)' <<< "${matrix}")"
test_matrix="$(sed -n 's/^test_matrix=//p' "${GITHUB_OUTPUT}")"
expected_controller_test_matrix="$(jq -c 'map({id, file, project}) | sort_by(.id)' <<< "${CONTROLLER_TEST_MATRIX}")"
actual_controller_test_matrix="$(jq -c 'map({id, file, project}) | sort_by(.id)' <<< "${test_matrix}")"
if [ "${actual_controller_matrix}" != "${expected_controller_matrix}" ] ||
[ "${actual_controller_test_matrix}" != "${expected_controller_test_matrix}" ]; then
: > "${GITHUB_OUTPUT}"
echo "::error::E2E planner matrix does not match controller-selected targets or shared tests" >&2
exit 1
fi
fi
native-runtime-qualification-producer-plan:
needs: generate-matrix
if: ${{ github.event_name == 'workflow_dispatch' && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha != '' && inputs.jobs == 'native-runtime-qualification-producer' && inputs.targets == '' }}
runs-on: ubuntu-latest
timeout-minutes: 10
permissions:
actions: read
contents: read
pull-requests: read
outputs:
matrix: ${{ steps.plan.outputs.matrix }}
steps:
- name: Check out the trusted qualification producer
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.workflow_sha }}
persist-credentials: false
sparse-checkout: |
src/lib/onboard/runtime-provider/native-qualification-authority.ts
test/e2e/registry/native-runtime-qualification.ts
tools/e2e/native-runtime-qualification-producer-plan.mts
sparse-checkout-cone-mode: false
- name: Check out the candidate for source authentication
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
path: .candidate-source
fetch-depth: 1
persist-credentials: false
- name: Set up Node for trusted qualification planning
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
- id: source
name: Authenticate the candidate and dispatch artifact
env:
BASE_SHA: ${{ inputs.base_sha }}
CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository }}
CANDIDATE_SHA: ${{ inputs.checkout_sha }}
GH_TOKEN: ${{ github.token }}
PR_NUMBER: ${{ inputs.pr_number }}
PRODUCER_RUN_ATTEMPT: ${{ github.run_attempt }}
PRODUCER_RUN_ID: ${{ github.run_id }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
[[ "$CANDIDATE_REPOSITORY" == "NVIDIA/NemoClaw" ]] || {
echo "::error::Native runtime qualification accepts only a branch in NVIDIA/NemoClaw" >&2
exit 1
}
[[ "$PRODUCER_RUN_ATTEMPT" == "1" ]] || {
echo "::error::Native runtime qualification producer runs cannot be rerun" >&2
exit 1
}
[[ "$BASE_SHA" == "$WORKFLOW_SHA" && "$CANDIDATE_SHA" != "$WORKFLOW_SHA" ]] || {
echo "::error::Native runtime qualification requires base_sha to equal workflow_sha and checkout_sha to name a different commit" >&2
exit 1
}
[[ "$(git -C .candidate-source rev-parse --verify 'HEAD^{commit}')" == "$CANDIDATE_SHA" ]] || {
echo "::error::Candidate checkout does not match the authenticated candidate commit" >&2
exit 1
}
pull="$(gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}")"
jq -e \
--arg baseSha "$BASE_SHA" \
--arg candidateRepository "$CANDIDATE_REPOSITORY" \
--arg candidateSha "$CANDIDATE_SHA" '
.state == "open" and
.head.repo.full_name == $candidateRepository and
.head.sha == $candidateSha and
.base.repo.full_name == "NVIDIA/NemoClaw" and
.base.ref == "main" and
.base.sha == $baseSha
' <<<"$pull" >/dev/null || {
echo "::error::Pull request identity changed before qualification planning" >&2
exit 1
}
artifact_name="e2e-dispatch-${PRODUCER_RUN_ID}-${PRODUCER_RUN_ATTEMPT}"
artifacts="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/runs/${PRODUCER_RUN_ID}/artifacts" \
-f name="$artifact_name" \
-f per_page=100)"
artifact="$(jq -ce \
--arg name "$artifact_name" \
--arg runId "$PRODUCER_RUN_ID" \
--arg workflowSha "$WORKFLOW_SHA" '
select(.total_count == 1 and (.artifacts | length) == 1) |
.artifacts[0] |
select(.name == $name and .expired == false) |
select(.size_in_bytes > 0 and .size_in_bytes <= 1048576) |
select((.id | tostring | test("^[1-9][0-9]{0,19}$"))) |
select((.digest // "") | test("^sha256:[a-f0-9]{64}$")) |
select((.workflow_run.id | tostring) == $runId) |
select(.workflow_run.head_sha == $workflowSha)
' <<<"$artifacts")" || {
echo "::error::Trusted dispatch artifact is missing or ambiguous" >&2
exit 1
}
installer_sha256="$(sha256sum .candidate-source/scripts/install.sh | awk '{print $1}')"
[[ "$installer_sha256" =~ ^[a-f0-9]{64}$ ]] || {
echo "::error::Candidate installer SHA-256 is invalid" >&2
exit 1
}
printf 'artifact_id=%s\n' "$(jq -r '.id | tostring' <<<"$artifact")" >>"$GITHUB_OUTPUT"
printf 'artifact_name=%s\n' "$artifact_name" >>"$GITHUB_OUTPUT"
printf 'artifact_digest=%s\n' "$(jq -r '.digest' <<<"$artifact")" >>"$GITHUB_OUTPUT"
printf 'artifact_size=%s\n' "$(jq -r '.size_in_bytes' <<<"$artifact")" >>"$GITHUB_OUTPUT"
printf 'installer_sha256=%s\n' "$installer_sha256" >>"$GITHUB_OUTPUT"
- id: plan
name: Compile the trusted qualification producer matrix
env:
BASE_SHA: ${{ inputs.base_sha }}
CANDIDATE_REPOSITORY: ${{ inputs.checkout_repository }}
CANDIDATE_SHA: ${{ inputs.checkout_sha }}
DISPATCH_ARTIFACT_DIGEST: ${{ steps.source.outputs.artifact_digest }}
DISPATCH_ARTIFACT_ID: ${{ steps.source.outputs.artifact_id }}
DISPATCH_ARTIFACT_NAME: ${{ steps.source.outputs.artifact_name }}
DISPATCH_ARTIFACT_SIZE: ${{ steps.source.outputs.artifact_size }}
INSTALLER_SHA256: ${{ steps.source.outputs.installer_sha256 }}
NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL: ${{ vars.NATIVE_RUNTIME_ARM64_GPU_RUNNER_LABEL }}
PR_NUMBER: ${{ inputs.pr_number }}
PRODUCER_RUN_ATTEMPT: ${{ github.run_attempt }}
PRODUCER_RUN_ID: ${{ github.run_id }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
run: node --experimental-strip-types --no-warnings tools/e2e/native-runtime-qualification-producer-plan.mts --ci-output
native-runtime-qualification-podman-toolchain:
name: Build pinned native Podman toolchain / ${{ matrix.architecture }}
needs: [generate-matrix, native-runtime-qualification-producer-plan]
if: ${{ needs.native-runtime-qualification-producer-plan.result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'native-runtime-qualification-producer') }}
strategy:
fail-fast: false
matrix:
include:
- architecture: amd64
runner: ubuntu-24.04
- architecture: arm64
runner: ubuntu-24.04-arm
runs-on: ${{ matrix.runner }}
timeout-minutes: 45
permissions:
contents: read
steps:
- name: Check out the pinned Podman source
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: podman-container-tools/podman
ref: cade97a52ebdf9dbf9e81de8009015776837a074 # v6.1.0
path: .podman-source
fetch-depth: 1
persist-credentials: false
- name: Check out the pinned Netavark source
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: containers/netavark
ref: 8e91ad1d947ed325327b638f0cb906bea1f7d0ab # v2.1.0
path: .netavark-source
fetch-depth: 1
persist-credentials: false
- name: Check out the pinned Aardvark DNS source
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: containers/aardvark-dns
ref: cd7417681229219059939bdd9f0b3bd9ac9abb08 # v2.1.0
path: .aardvark-source
fetch-depth: 1
persist-credentials: false
- name: Set up pinned Go for the Podman build
uses: actions/setup-go@44694675825211faa026b3c33043df3e48a5fa00 # v6.0.0
with:
go-version: 1.25.9
cache: false
- name: Set up pinned Rust for the network helper builds
uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1.17.0
with:
toolchain: 1.88.0
cache: false
rustflags: ""
- name: Install build dependencies from the signed runner OS repository
shell: bash
run: |
set -euo pipefail
sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update
sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install \
--yes --no-install-recommends \
curl gcc git libapparmor-dev libbtrfs-dev libc6-dev \
libdevmapper-dev libglib2.0-dev \
libprotobuf-c-dev libprotobuf-dev libseccomp-dev libselinux1-dev \
libsqlite3-dev libsystemd-dev make pkg-config protobuf-compiler
- name: Build and package the pinned native toolchain
env:
AARDVARK_SOURCE_SHA: cd7417681229219059939bdd9f0b3bd9ac9abb08
EXPECTED_ARCHITECTURE: ${{ matrix.architecture }}
NETAVARK_SOURCE_SHA: 8e91ad1d947ed325327b638f0cb906bea1f7d0ab
PASTA_SOURCE_ARCHIVE_SHA256: 54fc6a3b39b0fcb13182078662886a629032852e186e47a371fd9d7fd20d3958
PASTA_SOURCE_SHA: f8df3f1b228fe19a74a269334fdfe6cc7d0605ce
PASTA_VERSION: 2026_07_28.f8df3f1
PODMAN_SOURCE_SHA: cade97a52ebdf9dbf9e81de8009015776837a074
TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/native-runtime-podman-toolchain
shell: bash
run: |
set -euo pipefail
[[ "$(dpkg --print-architecture)" == "$EXPECTED_ARCHITECTURE" ]]
[[ "$(git -C .podman-source rev-parse --verify 'HEAD^{commit}')" == "$PODMAN_SOURCE_SHA" ]]
[[ "$(git -C .netavark-source rev-parse --verify 'HEAD^{commit}')" == "$NETAVARK_SOURCE_SHA" ]]
[[ "$(git -C .aardvark-source rev-parse --verify 'HEAD^{commit}')" == "$AARDVARK_SOURCE_SHA" ]]
[[ ! -e .passt-source && ! -L .passt-source ]]
pasta_source_archive="${RUNNER_TEMP}/passt-${PASTA_SOURCE_SHA}.tar.gz"
[[ ! -e "$pasta_source_archive" && ! -L "$pasta_source_archive" ]]
/usr/bin/curl \
--fail --location --proto '=https' --proto-redir '=https' \
--retry 3 --show-error --silent --tlsv1.2 \
--output "$pasta_source_archive" \
"https://passt.top/passt/snapshot/passt-${PASTA_SOURCE_SHA}.tar.gz"
[[ -f "$pasta_source_archive" && ! -L "$pasta_source_archive" ]]
[[ "$(sha256sum "$pasta_source_archive" | cut -d' ' -f1)" == "$PASTA_SOURCE_ARCHIVE_SHA256" ]]
mkdir .passt-source
tar \
--extract --gzip --file="$pasta_source_archive" \
--directory=.passt-source --strip-components=1 \
--no-same-owner --no-same-permissions
[[ -f .passt-source/Makefile && ! -L .passt-source/Makefile ]]
[[ -f .passt-source/passt.c && ! -L .passt-source/passt.c ]]
[[ ! -e .passt-source/passt && ! -L .passt-source/passt ]]
for source in .podman-source .netavark-source .aardvark-source; do
[[ -z "$(git -C "$source" status --porcelain --untracked-files=no)" ]]
done
[[ "$(go version)" == go\ version\ go1.25.9\ * ]]
[[ "$(rustc --version)" == rustc\ 1.88.0\ * ]]
SOURCE_DATE_EPOCH=1786554266 \
BUILD_ORIGIN="NVIDIA/NemoClaw native runtime qualification" \
EXTRA_BUILDTAGS=containers_image_openpgp \
make --directory=.podman-source --jobs=2 \
podman rootlessport PREFIX=/usr/local
SOURCE_DATE_EPOCH=1785940686 CI=1 \
make --directory=.netavark-source --jobs=2 build
SOURCE_DATE_EPOCH=1785940850 CI=1 \
make --directory=.aardvark-source --jobs=2 build
SOURCE_DATE_EPOCH=1785255008 \
make --directory=.passt-source --jobs=2 VERSION="$PASTA_VERSION" passt
podman_dependencies="$(ldd .podman-source/bin/podman)"
printf '%s\n' "$podman_dependencies"
if grep -F "not found" <<<"$podman_dependencies"; then
echo "::error::Pinned Podman build has an unresolved runtime dependency" >&2
exit 1
fi
if grep -E "libgpgme|libsubid" <<<"$podman_dependencies"; then
echo "::error::Pinned Podman build must not require an optional host ABI" >&2
exit 1
fi
install -D -m 0755 .podman-source/bin/podman "$TOOLCHAIN_DIRECTORY/bin/podman"
[[ -f .passt-source/passt && ! -L .passt-source/passt ]]
install -D -m 0755 .passt-source/passt "$TOOLCHAIN_DIRECTORY/bin/pasta"
install -D -m 0755 .podman-source/bin/rootlessport \
"$TOOLCHAIN_DIRECTORY/libexec/podman/rootlessport"
install -D -m 0755 .netavark-source/bin/netavark \
"$TOOLCHAIN_DIRECTORY/libexec/podman/netavark"
install -D -m 0755 .aardvark-source/bin/aardvark-dns \
"$TOOLCHAIN_DIRECTORY/libexec/podman/aardvark-dns"
install -D -m 0644 \
.podman-source/vendor/go.podman.io/common/pkg/config/containers.conf \
"$TOOLCHAIN_DIRECTORY/share/containers/containers.conf"
[[ "$("$TOOLCHAIN_DIRECTORY/bin/podman" --version)" == "podman version 6.1.0" ]]
pasta_version_output="$("$TOOLCHAIN_DIRECTORY/bin/pasta" --version)"
[[ "${pasta_version_output%%$'\n'*}" == "pasta $PASTA_VERSION" ]] || {
echo "::error::Pinned qualification pasta version is invalid" >&2
exit 1
}
[[ "$("$TOOLCHAIN_DIRECTORY/libexec/podman/netavark" --version)" == "netavark 2.1.0" ]]
[[ "$("$TOOLCHAIN_DIRECTORY/libexec/podman/aardvark-dns" --version)" == "aardvark-dns 2.1.0" ]]
jq -n \
--arg architecture "$EXPECTED_ARCHITECTURE" \
--arg aardvarkDnsSourceSha "$AARDVARK_SOURCE_SHA" \
--arg netavarkSourceSha "$NETAVARK_SOURCE_SHA" \
--arg pastaSourceArchiveSha256 "$PASTA_SOURCE_ARCHIVE_SHA256" \
--arg pastaSourceSha "$PASTA_SOURCE_SHA" \
--arg pastaVersion "$PASTA_VERSION" \
--arg podmanSourceSha "$PODMAN_SOURCE_SHA" '
{
schemaVersion: 1,
kind: "nemoclaw-native-podman-toolchain-v1",
architecture: $architecture,
podmanVersion: "6.1.0",
podmanSourceSha: $podmanSourceSha,
netavarkVersion: "2.1.0",
netavarkSourceSha: $netavarkSourceSha,
aardvarkDnsVersion: "2.1.0",
aardvarkDnsSourceSha: $aardvarkDnsSourceSha,
pastaVersion: $pastaVersion,
pastaSourceArchiveSha256: $pastaSourceArchiveSha256,
pastaSourceSha: $pastaSourceSha,
goVersion: "1.25.9",
rustVersion: "1.88.0"
}
' >"$TOOLCHAIN_DIRECTORY/manifest.json"
(
cd "$TOOLCHAIN_DIRECTORY"
sha256sum \
bin/pasta \
bin/podman \
libexec/podman/aardvark-dns \
libexec/podman/netavark \
libexec/podman/rootlessport \
manifest.json \
share/containers/containers.conf >SHA256SUMS
)
- name: Upload the pinned native Podman toolchain
if: success()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: native-runtime-podman-toolchain-${{ matrix.architecture }}
path: ${{ runner.temp }}/native-runtime-podman-toolchain/
native-runtime-qualification-producer:
name: ${{ matrix.jobName }}
needs:
- generate-matrix
- native-runtime-qualification-podman-toolchain
- native-runtime-qualification-producer-plan
if: ${{ needs.native-runtime-qualification-producer-plan.result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'native-runtime-qualification-producer') }}
strategy:
fail-fast: false
matrix: ${{ fromJSON(needs.native-runtime-qualification-producer-plan.outputs.matrix) }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 120
permissions:
contents: read
env:
NEMOCLAW_RUN_LIVE_E2E: "1"
QUALIFICATION_ROW: ${{ toJSON(matrix) }}
steps:
- name: Require an ephemeral qualification runner
env:
REVIEWED_EPHEMERAL_POOL: ${{ vars.NATIVE_RUNTIME_EPHEMERAL_RUNNER_POOL }}
RUNNER_ENVIRONMENT: ${{ runner.environment }}
shell: bash
run: |
set -euo pipefail
if [[ "$RUNNER_ENVIRONMENT" != "github-hosted" && "$REVIEWED_EPHEMERAL_POOL" != "enabled" ]]; then
echo "::error::Self-hosted native runtime qualification requires the reviewed ephemeral runner pool" >&2
exit 1
fi
- name: Check out the trusted qualification harness
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ matrix.source.workflowSha }}
path: .qualification-workflow
persist-credentials: false
sparse-checkout: |
src/lib/onboard/runtime-provider/native-qualification-authority.ts
scripts/checks/run-native-runtime-installer-qualification.sh
test/e2e/registry/native-runtime-qualification.ts
tools/e2e/native-runtime-qualification-producer-plan.mts
tools/e2e/native-runtime-qualification-producer-evidence.mts
sparse-checkout-cone-mode: false
- name: Check out the candidate commit
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ matrix.source.candidateRepository }}
ref: ${{ matrix.source.candidateSha }}
path: .candidate-runtime
fetch-depth: 1
persist-credentials: false
- name: Set up Node for candidate execution
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22.19.0
- name: Require a reviewed Ubuntu runtime host
shell: bash
run: |
set -euo pipefail
[[ -r /etc/os-release ]] || {
echo "::error::Protected runner does not expose an OS release identity" >&2
exit 1
}
# shellcheck disable=SC1091
source /etc/os-release
[[ "${ID:-}" == "ubuntu" ]] || {
echo "::error::Protected runner must use a reviewed Ubuntu image" >&2
exit 1
}
[[ "${VERSION_ID:-}" == "24.04" || "${VERSION_ID:-}" == "26.04" ]] || {
echo "::error::Protected runner Ubuntu release is not reviewed for Podman qualification" >&2
exit 1
}
- name: Download the pinned native Podman toolchain
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: native-runtime-podman-toolchain-${{ matrix.case.architecture }}
path: ${{ runner.temp }}/native-runtime-podman-toolchain
- name: Install the pinned native Podman toolchain and rootless prerequisites
env:
EXPECTED_ARCHITECTURE: ${{ matrix.case.architecture }}
TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/native-runtime-podman-toolchain
shell: bash
run: |
set -euo pipefail
[[ -x /usr/bin/apt-get ]] || {
echo "::error::Protected runner cannot install rootless Podman prerequisites from its signed OS repository" >&2
exit 1
}
sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get update
sudo env DEBIAN_FRONTEND=noninteractive /usr/bin/apt-get install \
--yes --no-install-recommends \
acl apparmor btrfs-progs conmon \
golang-github-containers-common iptables nftables runc slirp4netns uidmap
[[ -d "$TOOLCHAIN_DIRECTORY" && ! -L "$TOOLCHAIN_DIRECTORY" ]]
[[ -z "$(find -P "$TOOLCHAIN_DIRECTORY" -type l -print -quit)" ]]
mapfile -t actual_files < <(
cd "$TOOLCHAIN_DIRECTORY"
find . -type f -print | LC_ALL=C sort
)
expected_files=(
./SHA256SUMS
./bin/pasta
./bin/podman
./libexec/podman/aardvark-dns
./libexec/podman/netavark
./libexec/podman/rootlessport
./manifest.json
./share/containers/containers.conf
)
[[ "${actual_files[*]}" == "${expected_files[*]}" ]] || {
echo "::error::Downloaded native Podman toolchain contains unexpected files" >&2
exit 1
}
(
cd "$TOOLCHAIN_DIRECTORY"
sha256sum --check --strict SHA256SUMS
)
jq -e \
--arg architecture "$EXPECTED_ARCHITECTURE" '
type == "object" and
keys == [
"aardvarkDnsSourceSha",
"aardvarkDnsVersion",
"architecture",
"goVersion",
"kind",
"netavarkSourceSha",
"netavarkVersion",
"pastaSourceArchiveSha256",
"pastaSourceSha",
"pastaVersion",
"podmanSourceSha",
"podmanVersion",
"rustVersion",
"schemaVersion"
] and
.schemaVersion == 1 and
.kind == "nemoclaw-native-podman-toolchain-v1" and
.architecture == $architecture and
.podmanVersion == "6.1.0" and
.podmanSourceSha == "cade97a52ebdf9dbf9e81de8009015776837a074" and
.netavarkVersion == "2.1.0" and
.netavarkSourceSha == "8e91ad1d947ed325327b638f0cb906bea1f7d0ab" and
.aardvarkDnsVersion == "2.1.0" and
.aardvarkDnsSourceSha == "cd7417681229219059939bdd9f0b3bd9ac9abb08" and
.pastaVersion == "2026_07_28.f8df3f1" and
.pastaSourceArchiveSha256 == "54fc6a3b39b0fcb13182078662886a629032852e186e47a371fd9d7fd20d3958" and
.pastaSourceSha == "f8df3f1b228fe19a74a269334fdfe6cc7d0605ce" and
.goVersion == "1.25.9" and
.rustVersion == "1.88.0"
' "$TOOLCHAIN_DIRECTORY/manifest.json" >/dev/null
for target in \
/usr/local/bin/podman \
/usr/local/libexec/podman/aardvark-dns \
/usr/local/libexec/podman/netavark \
/usr/local/libexec/podman/rootlessport \
/usr/share/containers/containers.conf; do
[[ ! -L "$target" ]] || {
echo "::error::Native Podman toolchain target must not be a symlink: $target" >&2
exit 1
}
done
sudo install --owner=root --group=root --mode=0755 \
"$TOOLCHAIN_DIRECTORY/bin/podman" /usr/local/bin/podman
for helper in aardvark-dns netavark rootlessport; do
sudo install -D --owner=root --group=root --mode=0755 \
"$TOOLCHAIN_DIRECTORY/libexec/podman/$helper" \
"/usr/local/libexec/podman/$helper"
done
sudo install --owner=root --group=root --mode=0644 \
"$TOOLCHAIN_DIRECTORY/share/containers/containers.conf" \
/usr/share/containers/containers.conf
[[ "$(command -v podman)" == "/usr/local/bin/podman" ]]
conmon_version="$(conmon --version | awk 'NR == 1 { print $NF }')"
runc_version="$(runc --version | awk 'NR == 1 { print $NF }')"
dpkg --compare-versions "$conmon_version" ge 2.1.7
dpkg --compare-versions "$runc_version" ge 1.1.11
[[ "$(/usr/local/libexec/podman/netavark --version)" == "netavark 2.1.0" ]]
[[ "$(/usr/local/libexec/podman/aardvark-dns --version)" == "aardvark-dns 2.1.0" ]]
version="$(podman --version)"
[[ "$version" == "podman version 6.1.0" ]]
- name: Prepare the credential-free execution account and disable Docker
id: boundary
env:
CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime
TOOLCHAIN_DIRECTORY: ${{ runner.temp }}/native-runtime-podman-toolchain
shell: bash
run: |
set -euo pipefail
for command in apparmor_parser awk cat curl getent git grep groupdel id jq node npm pgrep podman setfacl sha256sum stat systemctl tee unlink useradd userdel usermod; do
command -v "$command" >/dev/null || {
echo "::error::Protected runner is missing required command: $command" >&2
exit 1
}
done
[[ "$(podman --version)" == "podman version 6.1.0" ]] || {
echo "::error::Protected runner must provide Podman 6.1.0" >&2
exit 1
}
sudo systemctl stop docker.service docker.socket 2>/dev/null || true
sudo systemctl mask --runtime docker.service docker.socket 2>/dev/null || true
sudo pkill -TERM -x dockerd 2>/dev/null || true
sudo rm -f /var/run/docker.sock /run/docker.sock
! systemctl is-active --quiet docker.service
! systemctl is-active --quiet docker.socket
! pgrep -x dockerd >/dev/null
[[ ! -S /var/run/docker.sock && ! -S /run/docker.sock ]]
account="nemoclawq"
ownership_marker="/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
if getent passwd "$account" >/dev/null || getent group "$account" >/dev/null || grep -q "^${account}:" /etc/subuid /etc/subgid; then
echo "::error::Qualification account, group, or subordinate-ID authorization already exists" >&2
exit 1
fi
[[ ! -e "$ownership_marker" && ! -L "$ownership_marker" ]] || {
echo "::error::Qualification account ownership marker already exists" >&2
exit 1
}
account_created_without_marker=0
rollback_unmarked_account() {
local result="$?"
trap - EXIT
if ((result != 0 && account_created_without_marker == 1)); then
sudo userdel --remove "$account" 2>/dev/null || true
sudo groupdel "$account" 2>/dev/null || true
sudo rm -f -- "$ownership_marker"
if getent passwd "$account" >/dev/null || getent group "$account" >/dev/null || grep -q "^${account}:" /etc/subuid /etc/subgid; then
echo "::error::Partially created qualification account could not be rolled back" >&2
exit 1
fi
fi
exit "$result"
}
trap rollback_unmarked_account EXIT
sudo useradd --create-home --shell /usr/sbin/nologin --user-group "$account"
account_created_without_marker=1
uid="$(id -u "$account")"
gid="$(id -g "$account")"
home="$(getent passwd "$account" | cut -d: -f6)"
group_entry="$(getent group "$account")"
[[ "$uid" =~ ^[0-9]+$ && "$gid" =~ ^[0-9]+$ && "$home" == "/home/${account}" && -d "$home" && ! -L "$home" ]] || {
echo "::error::Qualification account identity is missing or invalid" >&2
exit 1
}
[[ "$group_entry" == "${account}:x:${gid}:" ]] || {
echo "::error::Qualification private group identity is missing or invalid" >&2
exit 1
}
printf '%s:%s:%s\n' "$account" "$uid" "$gid" | sudo tee "$ownership_marker" >/dev/null
sudo chown root:root "$ownership_marker"
sudo chmod 0400 "$ownership_marker"
[[ -f "$ownership_marker" && ! -L "$ownership_marker" && "$(stat -c '%u:%g:%a' "$ownership_marker")" == "0:0:400" ]] || {
echo "::error::Qualification account ownership marker is invalid" >&2
exit 1
}
account_created_without_marker=0
trap - EXIT
ensure_subordinate_range() {
local file="$1"
local option="$2"
local range_start=100000
local range_end
local conflict_end
[[ -f "$file" && ! -L "$file" ]] || {
echo "::error::Rootless Podman subordinate-ID file is missing or invalid: $file" >&2
exit 1
}
if awk -F: -v account="$account" '
$1 == account && $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ && $3 >= 65536 { found = 1 }
END { exit found ? 0 : 1 }
' "$file"; then
return
fi
while :; do
((range_start <= 4294901760)) || {
echo "::error::Protected runner has no free subordinate-ID range for rootless Podman" >&2
exit 1
}
range_end=$((range_start + 65535))
conflict_end="$(awk -F: -v start="$range_start" -v end="$range_end" '
$2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ {
current_end = $2 + $3 - 1
if ($2 <= end && current_end >= start && current_end > maximum) maximum = current_end
}
END { if (maximum != "") print maximum }
' "$file")"
[[ -n "$conflict_end" ]] || break
range_start=$((conflict_end + 1))
done
range_end=$((range_start + 65535))
sudo usermod "$option" "${range_start}-${range_end}" "$account"
awk -F: -v account="$account" '
$1 == account && $2 ~ /^[0-9]+$/ && $3 ~ /^[0-9]+$/ && $3 >= 65536 { found = 1 }
END { exit found ? 0 : 1 }
' "$file" || {
echo "::error::Protected runner did not provision rootless Podman subordinate IDs" >&2
exit 1
}
}
ensure_subordinate_range /etc/subuid --add-subuids
ensure_subordinate_range /etc/subgid --add-subgids
printf 'account=%s\n' "$account" >>"$GITHUB_OUTPUT"
printf 'account_created=true\n' >>"$GITHUB_OUTPUT"
printf 'uid=%s\n' "$uid" >>"$GITHUB_OUTPUT"
printf 'gid=%s\n' "$gid" >>"$GITHUB_OUTPUT"
runtime_dir="/run/user/${uid}"
runtime_directory_unit="user-runtime-dir@${uid}.service"
user_manager_unit="user@${uid}.service"
user_manager_dropin_directory="/run/systemd/system/${user_manager_unit}.d"
user_manager_dropin="${user_manager_dropin_directory}/50-nemoclaw-native-runtime.conf"
trusted_user_unit_path="/usr/lib/systemd/user:/lib/systemd/user"
[[ ! -e "$user_manager_dropin_directory" && ! -L "$user_manager_dropin_directory" ]] || {
echo "::error::Qualification systemd user-manager drop-in directory already exists" >&2
exit 1
}
sudo install -d --owner=root --group=root --mode=0755 "$user_manager_dropin_directory"
printf '[Service]\nEnvironment="SYSTEMD_UNIT_PATH=%s"\n' "$trusted_user_unit_path" |
sudo tee "$user_manager_dropin" >/dev/null
sudo chown root:root "$user_manager_dropin"
sudo chmod 0444 "$user_manager_dropin"
[[ -f "$user_manager_dropin" && ! -L "$user_manager_dropin" && "$(stat -c '%u:%g:%a:%h' "$user_manager_dropin")" == "0:0:444:1" ]] || {
echo "::error::Qualification systemd user-manager drop-in is invalid" >&2
exit 1
}
sudo systemctl daemon-reload
verify_user_manager_unit_path() {
local environment
environment="$(sudo -u "$1" env -i \
HOME="$2" \
LANG=C.UTF-8 \
PATH=/usr/bin:/bin \
XDG_RUNTIME_DIR="$3" \
/usr/bin/systemctl --user show-environment)"
tr ' ' '\n' <<<"$environment" | grep -Fx -- "SYSTEMD_UNIT_PATH=$trusted_user_unit_path" >/dev/null || {
echo "::error::Qualification systemd user manager did not inherit the trusted unit path" >&2
return 1
}
}
verify_user_bus() {
local execution_account="$1"
local expected_uid="$2"
local bus="$3"
local context="$4"
sudo /usr/bin/test -S "$bus" &&
sudo /usr/bin/test ! -L "$bus" &&
[[ "$(sudo stat -c '%u' -- "$bus")" == "$expected_uid" ]] &&
sudo -u "$execution_account" /usr/bin/test -S "$bus" || {
echo "::error::Qualification systemd user bus $context" >&2
return 1
}
}
sudo systemctl start "$user_manager_unit"
systemctl is-active --quiet "$runtime_directory_unit" || {
echo "::error::Qualification systemd runtime-directory unit is not active" >&2
exit 1
}
[[ -d "$runtime_dir" && ! -L "$runtime_dir" && "$(stat -c '%u:%g:%a' "$runtime_dir")" == "${uid}:${gid}:700" ]] || {
echo "::error::Qualification runtime directory is missing or invalid" >&2
exit 1
}
systemctl is-active --quiet "$user_manager_unit" || {
echo "::error::Qualification systemd user manager is not active" >&2
exit 1
}
sudo -u "$account" env -i \
HOME="$home" \
LANG=C.UTF-8 \
PATH=/usr/bin:/bin \
XDG_RUNTIME_DIR="$runtime_dir" \
/usr/bin/systemctl --user start dbus.socket
sudo -u "$account" env -i \
HOME="$home" \
LANG=C.UTF-8 \
PATH=/usr/bin:/bin \
XDG_RUNTIME_DIR="$runtime_dir" \
/usr/bin/systemctl --user is-active --quiet dbus.socket || {
echo "::error::Qualification systemd user bus socket unit is not active" >&2
exit 1
}
verify_user_bus "$account" "$uid" "$runtime_dir/bus" \
"is invalid or inaccessible to the execution account"
verify_user_manager_unit_path "$account" "$home" "$runtime_dir"
storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
storage_config="${storage_config_directory}/storage.conf"
containers_config="${storage_config_directory}/containers.conf"
podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
apparmor_profile="${storage_config_directory}/podman.apparmor"
apparmor_profile_name="nemoclaw-native-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
helper_directory="/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
pasta_executable="${helper_directory}/pasta"
pasta_apparmor_profile="${storage_config_directory}/pasta.apparmor"
pasta_apparmor_profile_name="nemoclaw-native-pasta-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
[[ ! -e "$storage_config_directory" && ! -L "$storage_config_directory" ]] || {
echo "::error::Qualification storage configuration directory already exists" >&2
exit 1
}
[[ ! -e "$helper_directory" && ! -L "$helper_directory" ]] || {
echo "::error::Qualification helper directory already exists" >&2
exit 1
}
sudo install -d -o root -g root -m 0755 "$storage_config_directory"
[[ -f "$TOOLCHAIN_DIRECTORY/bin/podman" && ! -L "$TOOLCHAIN_DIRECTORY/bin/podman" ]] || {
echo "::error::Pinned qualification Podman executable source is missing or invalid" >&2
exit 1
}
[[ ! -e "$podman_executable" && ! -L "$podman_executable" ]] || {
echo "::error::Run-owned qualification Podman executable already exists" >&2
exit 1
}
sudo install --owner=root --group=root --mode=0555 \
"$TOOLCHAIN_DIRECTORY/bin/podman" "$podman_executable"
[[ -f "$podman_executable" && ! -L "$podman_executable" && "$(stat -c '%u:%g:%a' "$podman_executable")" == "0:0:555" ]] || {
echo "::error::Run-owned qualification Podman executable is invalid" >&2
exit 1
}
[[ "$(sha256sum "$podman_executable" | cut -d' ' -f1)" == "$(sha256sum "$TOOLCHAIN_DIRECTORY/bin/podman" | cut -d' ' -f1)" ]] || {
echo "::error::Run-owned qualification Podman executable digest changed during installation" >&2
exit 1
}
[[ -f "$TOOLCHAIN_DIRECTORY/bin/pasta" && ! -L "$TOOLCHAIN_DIRECTORY/bin/pasta" ]] || {
echo "::error::Pinned qualification pasta executable source is missing or invalid" >&2
exit 1
}
sudo install -d --owner=root --group=root --mode=0555 "$helper_directory"
sudo install --owner=root --group=root --mode=0555 \
"$TOOLCHAIN_DIRECTORY/bin/pasta" "$pasta_executable"
[[ -d "$helper_directory" && ! -L "$helper_directory" && "$(stat -c '%u:%g:%a' "$helper_directory")" == "0:0:555" ]] || {
echo "::error::Run-owned qualification helper directory is invalid" >&2
exit 1
}
[[ -f "$pasta_executable" && ! -L "$pasta_executable" && "$(stat -c '%u:%g:%a' "$pasta_executable")" == "0:0:555" ]] || {
echo "::error::Run-owned qualification pasta executable is invalid" >&2
exit 1
}
[[ "$(sha256sum "$pasta_executable" | cut -d' ' -f1)" == "$(sha256sum "$TOOLCHAIN_DIRECTORY/bin/pasta" | cut -d' ' -f1)" ]] || {
echo "::error::Run-owned qualification pasta executable digest changed during installation" >&2
exit 1
}
printf '%s\n' \
'[storage]' \
'driver = "overlay"' \
"runroot = \"${home}/.local/share/containers/runroot\"" \
"graphroot = \"${home}/.local/share/containers/storage\"" \
"rootless_storage_path = \"${home}/.local/share/containers/storage\"" | sudo tee "$storage_config" >/dev/null
sudo chown root:root "$storage_config"
sudo chmod 0444 "$storage_config"
[[ -f "$storage_config" && ! -L "$storage_config" && "$(stat -c '%u:%g:%a' "$storage_config")" == "0:0:444" ]] || {
echo "::error::Qualification storage configuration is not root-owned and read-only" >&2
exit 1
}
printf '%s\n' \
'[network]' \
'firewall_driver = "nftables"' | sudo tee "$containers_config" >/dev/null
sudo chown root:root "$containers_config"
sudo chmod 0444 "$containers_config"
[[ -f "$containers_config" && ! -L "$containers_config" && "$(stat -c '%u:%g:%a' "$containers_config")" == "0:0:444" ]] || {
echo "::error::Qualification containers configuration is not root-owned and read-only" >&2
exit 1
}
if [[ -r /sys/module/apparmor/parameters/enabled ]] && grep -q '^Y' /sys/module/apparmor/parameters/enabled; then
printf '%s\n' \
'# This ephemeral profile grants user namespaces only to the pinned qualification Podman binary.' \
'' \
'abi <abi/4.0>,' \
'include <tunables/global>' \
'' \
"profile ${apparmor_profile_name} ${podman_executable} flags=(unconfined) {" \
' userns,' \
'}' | sudo tee "$apparmor_profile" >/dev/null
sudo chown root:root "$apparmor_profile"
sudo chmod 0444 "$apparmor_profile"
[[ -f "$apparmor_profile" && ! -L "$apparmor_profile" && "$(stat -c '%u:%g:%a' "$apparmor_profile")" == "0:0:444" ]] || {
echo "::error::Qualification AppArmor profile is not root-owned and read-only" >&2
exit 1
}
sudo apparmor_parser -r "$apparmor_profile"
printf '%s\n' \
'# This ephemeral profile is limited to the immutable run-owned pasta helper.' \
'' \
'abi <abi/4.0>,' \
'include <tunables/global>' \
'' \
"profile ${pasta_apparmor_profile_name} ${pasta_executable} flags=(unconfined) {" \
' userns,' \
'}' | sudo tee "$pasta_apparmor_profile" >/dev/null
sudo chown root:root "$pasta_apparmor_profile"
sudo chmod 0444 "$pasta_apparmor_profile"
[[ -f "$pasta_apparmor_profile" && ! -L "$pasta_apparmor_profile" && "$(stat -c '%u:%g:%a' "$pasta_apparmor_profile")" == "0:0:444" ]] || {
echo "::error::Qualification pasta AppArmor profile is not root-owned and read-only" >&2
exit 1
}
sudo apparmor_parser -r "$pasta_apparmor_profile"
fi
ancestor="$(dirname "$CANDIDATE_DIRECTORY")"
while [[ "$ancestor" != "/home" ]]; do
[[ ("$ancestor" == "/home/runner" || "$ancestor" == /home/runner/*) && -d "$ancestor" && ! -L "$ancestor" ]] || {
echo "::error::Candidate checkout ancestor is outside the reviewed runner workspace" >&2
exit 1
}
sudo setfacl --modify "u:${account}:--x" "$ancestor"
ancestor="$(dirname "$ancestor")"
done
sudo chown -R "$uid:$gid" "$CANDIDATE_DIRECTORY"
node_directory="$(dirname "$(command -v node)")"
[[ "$node_directory" == /* && -x "$node_directory/node" && -x "$node_directory/npm" ]] || {
echo "::error::Pinned Node toolchain path is invalid" >&2
exit 1
}
guard_dir="${RUNNER_TEMP}/native-runtime-docker-guard"
install -d -m 0755 "$guard_dir"
printf '%s\n' '#!/usr/bin/env bash' 'exit 97' >"$guard_dir/docker"
chmod 0555 "$guard_dir/docker"
podman_info="$(sudo -u "$account" env -i \
CONTAINERS_CONF="$containers_config" \
CONTAINERS_STORAGE_CONF="$storage_config" \
HOME="$home" \
LANG=C.UTF-8 \
PATH="$guard_dir:$helper_directory:/usr/local/bin:/usr/bin:/bin" \
XDG_RUNTIME_DIR="$runtime_dir" \
"$podman_executable" info --format json)" || {
echo "::error::Credential-free rootless Podman readiness failed" >&2
exit 1
}
jq -e '
.host.security.rootless == true and
.store.graphDriverName == "overlay" and
(((.store.graphOptions // {})["overlay.mount_program"].Executable? // "") == "")
' <<<"$podman_info" >/dev/null || {
echo "::error::Qualification requires native rootless overlay storage" >&2
exit 1
}
printf 'home=%s\n' "$home" >>"$GITHUB_OUTPUT"
printf 'runtime_dir=%s\n' "$runtime_dir" >>"$GITHUB_OUTPUT"
printf 'guard_dir=%s\n' "$guard_dir" >>"$GITHUB_OUTPUT"
printf 'helper_dir=%s\n' "$helper_directory" >>"$GITHUB_OUTPUT"
printf 'node_dir=%s\n' "$node_directory" >>"$GITHUB_OUTPUT"
printf 'podman_executable=%s\n' "$podman_executable" >>"$GITHUB_OUTPUT"
printf 'containers_config=%s\n' "$containers_config" >>"$GITHUB_OUTPUT"
printf 'runtime_directory_unit=%s\n' "$runtime_directory_unit" >>"$GITHUB_OUTPUT"
printf 'storage_config=%s\n' "$storage_config" >>"$GITHUB_OUTPUT"
printf 'user_manager_unit=%s\n' "$user_manager_unit" >>"$GITHUB_OUTPUT"
- name: Prepare GPU resources with the NVIDIA API key
id: gpu_resources
env:
ACCOUNT: ${{ steps.boundary.outputs.account }}
ACCOUNT_GID: ${{ steps.boundary.outputs.gid }}
ACCOUNT_UID: ${{ steps.boundary.outputs.uid }}
ACCELERATION: ${{ matrix.case.acceleration }}
ARCHITECTURE: ${{ matrix.case.architecture }}
CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }}
GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }}
HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }}
INFERENCE: ${{ matrix.case.inference }}
NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
PODMAN_EXECUTABLE: ${{ steps.boundary.outputs.podman_executable }}
QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }}
RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }}
STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }}
shell: bash
run: |
set -euo pipefail
umask 077
if [[ "$ACCELERATION" != "nvidia-gpu" ]]; then
printf 'runner_contract=\n' >>"$GITHUB_OUTPUT"
exit 0
fi
[[ -n "$NVIDIA_API_KEY" ]] || {
echo "::error::Native runtime GPU preparation requires the NVIDIA_API_KEY repository secret" >&2
exit 1
}
uid="$(id -u "$ACCOUNT")"
gid="$(id -g "$ACCOUNT")"
[[ "$uid" == "$ACCOUNT_UID" && "$gid" == "$ACCOUNT_GID" ]] || {
echo "::error::Qualification account identity changed before GPU resource preparation" >&2
exit 1
}
storage_config_directory="$(dirname "$STORAGE_CONFIG")"
runner_contract="${storage_config_directory}/runner-contract.json"
registry_auth_directory="${storage_config_directory}/registry-auth"
registry_auth_file="${registry_auth_directory}/auth.json"
resource_directory="/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
model_directory="${resource_directory}/model"
model_revision="7ae557604adf67be50417f59c2c2f167def9a775"
model_name="Qwen/Qwen2.5-0.5B-Instruct"
case "$ARCHITECTURE" in
amd64)
probe_image="nvcr.io/nvidia/k8s/cuda-sample@sha256:9855f4c8500addf185360474184b9efdaf4384284779aa9173dcd70164a4ae6f"
nim_image="nvcr.io/nim/nvidia/model-free-nim@sha256:a0fdbecdf51792dadc48d284fde3199a58d5a2067007ad5b80319975fe81ce93"
vllm_image="nvcr.io/nvidia/vllm@sha256:7be6c2f676c36059a494fe17254e69ae5c677535ba6191044e5fc8e42a91c773"
;;
arm64)
probe_image="nvcr.io/nvidia/k8s/cuda-sample@sha256:a54fdceac3bc2a8d177f07db942defc2f7237e18d07fca2ff00718ba5aee4940"
nim_image="nvcr.io/nim/nvidia/model-free-nim@sha256:8342257b9744e9bc23a02e0f45badad6b88474727473965b4dc83e8fee45956a"
vllm_image="nvcr.io/nvidia/vllm@sha256:447995cbb57e6c7cf792cab95e9852e5f62b5fb6d2f39e030fa4eda9a54eadb4"
;;
*)
echo "::error::GPU resource architecture is unsupported" >&2
exit 1
;;
esac
[[ ! -e "$runner_contract" && ! -L "$runner_contract" ]] || {
echo "::error::Run-owned GPU runner contract already exists" >&2
exit 1
}
[[ ! -e "$registry_auth_directory" && ! -L "$registry_auth_directory" ]] || {
echo "::error::Run-owned registry authentication directory already exists" >&2
exit 1
}
sudo install -d --owner="$uid" --group="$gid" --mode=0700 "$registry_auth_directory"
cleanup_registry_auth() {
local result="$?"
trap - EXIT
if sudo test -e "$registry_auth_file" || sudo test -L "$registry_auth_file"; then
sudo test -f "$registry_auth_file" &&
sudo test ! -L "$registry_auth_file" &&
[[ "$(sudo stat -c '%u:%g:%a:%h' -- "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || {
echo "::error::Run-owned registry authentication file cleanup target is invalid" >&2
return 1
}
sudo unlink "$registry_auth_file"
fi
sudo rmdir "$registry_auth_directory" || {
echo "::error::Run-owned registry authentication directory was not empty" >&2
return 1
}
return "$result"
}
trap cleanup_registry_auth EXIT
printf '%s' "$NVIDIA_API_KEY" | sudo -u "$ACCOUNT" env -i \
CONTAINERS_CONF="$CONTAINERS_CONFIG" \
CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
PATH="$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \
REGISTRY_AUTH_FILE="$registry_auth_file" \
XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \
"$PODMAN_EXECUTABLE" login nvcr.io --username '$oauthtoken' --password-stdin >/dev/null
sudo test -f "$registry_auth_file" &&
sudo test ! -L "$registry_auth_file" &&
[[ "$(sudo stat -c '%u:%g:%h' -- "$registry_auth_file")" == "${uid}:${gid}:1" ]] || {
echo "::error::Run-owned registry authentication file is invalid" >&2
exit 1
}
sudo chmod 0600 -- "$registry_auth_file"
[[ "$(sudo stat -c '%u:%g:%a:%h' -- "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || {
echo "::error::Run-owned registry authentication file permissions are invalid" >&2
exit 1
}
image_to_pull="$probe_image"
if [[ "$INFERENCE" == "nim" ]]; then
image_to_pull="$nim_image"
elif [[ "$INFERENCE" == "vllm" ]]; then
image_to_pull="$vllm_image"
fi
images=("$probe_image")
[[ "$image_to_pull" == "$probe_image" ]] || images+=("$image_to_pull")
for image in "${images[@]}"; do
sudo -u "$ACCOUNT" env -i \
CONTAINERS_CONF="$CONTAINERS_CONFIG" \
CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \
REGISTRY_AUTH_FILE="$registry_auth_file" \
XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \
"$PODMAN_EXECUTABLE" pull "$image"
done
cleanup_registry_auth
trap - EXIT
unset NVIDIA_API_KEY
if [[ "$INFERENCE" == "nim" || "$INFERENCE" == "vllm" ]]; then
[[ ! -e "$resource_directory" && ! -L "$resource_directory" ]] || {
echo "::error::Run-owned GPU model resource already exists" >&2
exit 1
}
sudo install -d --owner=root --group=root --mode=0711 "$resource_directory"
sudo install -d --owner="$uid" --group="$gid" --mode=0700 "$model_directory"
download_model_file() {
local file="$1"
local size="$2"
local algorithm="$3"
local digest="$4"
local target="${model_directory}/${file}"
sudo -u "$ACCOUNT" env -i \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
/usr/bin/curl \
--fail --location --proto '=https' --retry 3 --show-error --silent --tlsv1.2 \
--output "$target" \
"https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct/resolve/${model_revision}/${file}?download=true"
sudo test -f "$target" &&
sudo test ! -L "$target" &&
[[ "$(sudo stat -c '%u:%g:%h:%s' -- "$target")" == "${uid}:${gid}:1:${size}" ]] || {
echo "::error::Downloaded GPU model file metadata is invalid: $file" >&2
exit 1
}
sudo chmod 0600 -- "$target"
[[ "$(sudo stat -c '%u:%g:%a:%h:%s' -- "$target")" == "${uid}:${gid}:600:1:${size}" ]] || {
echo "::error::Downloaded GPU model file permissions are invalid: $file" >&2
exit 1
}
if [[ "$algorithm" == "sha256" ]]; then
[[ "$(sudo sha256sum -- "$target" | cut -d' ' -f1)" == "$digest" ]]
else
[[ "$(sudo git hash-object --no-filters -- "$target")" == "$digest" ]]
fi || {
echo "::error::Downloaded GPU model file digest is invalid: $file" >&2
exit 1
}
}
download_model_file config.json 659 sha1 0dbb161213629a23f0fc00ef286e6b1e366d180f
download_model_file generation_config.json 242 sha1 dfc11073787daf1b0f9c0f1499487ab5f4c93738
download_model_file merges.txt 1671839 sha1 20024bfe7c83998e9aeaf98a0cd6a2ce6306c2f0
download_model_file model.safetensors 988097824 sha256 fdf756fa7fcbe7404d5c60e26bff1a0c8b8aa1f72ced49e7dd0210fe288fb7fe
download_model_file tokenizer.json 7031645 sha1 443909a61d429dff23010e5bddd28ff530edda00
download_model_file tokenizer_config.json 7305 sha1 07bfe0640cb5a0037f9322287fbfc682806cf672
download_model_file vocab.json 2776833 sha1 4783fe10ac3adce15ac8f358ef5462739852c569
for file in config.json generation_config.json merges.txt model.safetensors tokenizer.json tokenizer_config.json vocab.json; do
sudo chown root:root "${model_directory}/${file}"
sudo chmod 0444 "${model_directory}/${file}"
done
sudo chown root:root "$model_directory"
sudo chmod 0555 "$model_directory"
sudo chmod 0555 "$resource_directory"
fi
jq -n \
--arg architecture "$ARCHITECTURE" \
--arg gpuProbeImageRef "$probe_image" \
--arg model "$model_name" \
--arg modelPath "$model_directory" \
--arg modelRevision "$model_revision" \
--arg nimImageRef "$nim_image" \
--arg vllmImageRef "$vllm_image" '
{
schemaVersion: 1,
kind: "nemoclaw-native-runtime-qualification-runner-v1",
architecture: $architecture,
gpuProbeImageRef: $gpuProbeImageRef,
nim: {
imageRef: $nimImageRef,
model: $model,
modelPath: $modelPath,
modelRevision: $modelRevision
},
vllm: {
imageRef: $vllmImageRef,
model: $model,
modelPath: $modelPath,
modelRevision: $modelRevision
}
}
' | sudo tee "$runner_contract" >/dev/null
sudo chown root:root "$runner_contract"
sudo chmod 0444 "$runner_contract"
[[ -f "$runner_contract" && ! -L "$runner_contract" && "$(stat -c '%u:%g:%a' "$runner_contract")" == "0:0:444" ]] || {
echo "::error::Run-owned GPU runner contract is invalid" >&2
exit 1
}
printf 'runner_contract=%s\n' "$runner_contract" >>"$GITHUB_OUTPUT"
- name: Install locked candidate test dependencies without scripts
env:
ACCOUNT: ${{ steps.boundary.outputs.account }}
CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime
GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }}
HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }}
NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }}
QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }}
shell: bash
run: |
set -euo pipefail
sudo -u "$ACCOUNT" env -i \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \
/bin/bash --noprofile --norc -c '
set -euo pipefail
cd "$1"
for file in package.json package-lock.json; do
[[ -f "$file" && ! -L "$file" && -O "$file" ]] || {
echo "Candidate dependency manifest is missing or invalid: $file" >&2
exit 1
}
done
exec npm --prefix "$1" ci --ignore-scripts
' bash "$CANDIDATE_DIRECTORY"
- name: Run the authenticated installer qualification
env:
ACCOUNT: ${{ steps.boundary.outputs.account }}
ACCOUNT_GID: ${{ steps.boundary.outputs.gid }}
ACCOUNT_UID: ${{ steps.boundary.outputs.uid }}
ARCHITECTURE: ${{ matrix.case.architecture }}
CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime
CANDIDATE_SHA: ${{ matrix.source.candidateSha }}
CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }}
GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }}
HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }}
INSTALLER_RECEIPT_PARENT: ${{ runner.temp }}/native-runtime-installer
INSTALLER_SHA256: ${{ matrix.installerSha256 }}
QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }}
RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }}
RUNTIME_DIRECTORY_UNIT: ${{ steps.boundary.outputs.runtime_directory_unit }}
STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }}
TRUSTED_USER_UNIT_PATH: /usr/lib/systemd/user:/lib/systemd/user
USER_MANAGER_UNIT: ${{ steps.boundary.outputs.user_manager_unit }}
shell: bash
run: |
set -euo pipefail
install -d -m 0700 "$INSTALLER_RECEIPT_PARENT"
sudo chown "$ACCOUNT_UID:$ACCOUNT_GID" "$INSTALLER_RECEIPT_PARENT"
sudo -u "$ACCOUNT" env -i \
CONTAINERS_CONF="$CONTAINERS_CONFIG" \
CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \
XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \
bash .qualification-workflow/scripts/checks/run-native-runtime-installer-qualification.sh \
--candidate-checkout "$CANDIDATE_DIRECTORY" \
--candidate-sha "$CANDIDATE_SHA" \
--installer-sha256 "$INSTALLER_SHA256" \
--architecture "$ARCHITECTURE" \
--artifact-dir "$INSTALLER_RECEIPT_PARENT/receipts"
sudo systemctl stop "$USER_MANAGER_UNIT" "$RUNTIME_DIRECTORY_UNIT"
! systemctl is-active --quiet "$USER_MANAGER_UNIT" && ! systemctl is-active --quiet "$RUNTIME_DIRECTORY_UNIT" || {
echo "::error::Qualification systemd user lifecycle remained active after installer isolation" >&2
exit 1
}
sudo pkill -KILL -u "$ACCOUNT_UID" 2>/dev/null || true
sudo systemctl start "$USER_MANAGER_UNIT"
systemctl is-active --quiet "$USER_MANAGER_UNIT" && systemctl is-active --quiet "$RUNTIME_DIRECTORY_UNIT" || {
echo "::error::Qualification systemd user lifecycle did not restart after installer isolation" >&2
exit 1
}
sudo -u "$ACCOUNT" env -i \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
PATH=/usr/bin:/bin \
XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \
/usr/bin/systemctl --user start dbus.socket
sudo -u "$ACCOUNT" env -i \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
PATH=/usr/bin:/bin \
XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \
/usr/bin/systemctl --user is-active --quiet dbus.socket || {
echo "::error::Qualification systemd user bus socket unit did not restart after installer isolation" >&2
exit 1
}
sudo /usr/bin/test -S "$RUNTIME_DIRECTORY/bus" &&
sudo /usr/bin/test ! -L "$RUNTIME_DIRECTORY/bus" &&
[[ "$(sudo stat -c '%u' -- "$RUNTIME_DIRECTORY/bus")" == "$ACCOUNT_UID" ]] &&
sudo -u "$ACCOUNT" /usr/bin/test -S "$RUNTIME_DIRECTORY/bus" || {
echo "::error::Qualification systemd user bus is invalid or inaccessible after installer isolation" >&2
exit 1
}
manager_environment="$(sudo -u "$ACCOUNT" env -i \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
PATH=/usr/bin:/bin \
XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \
/usr/bin/systemctl --user show-environment)"
tr ' ' '\n' <<<"$manager_environment" | grep -Fx -- "SYSTEMD_UNIT_PATH=$TRUSTED_USER_UNIT_PATH" >/dev/null || {
echo "::error::Qualification systemd user manager lost the trusted unit path after installer isolation" >&2
exit 1
}
sudo test -d "$INSTALLER_RECEIPT_PARENT/receipts" || {
echo "::error::Installer receipt directory is missing or invalid" >&2
exit 1
}
sudo test ! -L "$INSTALLER_RECEIPT_PARENT/receipts" || {
echo "::error::Installer receipt directory must not be a symlink" >&2
exit 1
}
- name: Execute the candidate qualification case without credentials
env:
ACCOUNT: ${{ steps.boundary.outputs.account }}
ACCOUNT_GID: ${{ steps.boundary.outputs.gid }}
ACCOUNT_UID: ${{ steps.boundary.outputs.uid }}
CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime
CONTAINERS_CONFIG: ${{ steps.boundary.outputs.containers_config }}
GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }}
HELPER_DIRECTORY: ${{ steps.boundary.outputs.helper_dir }}
NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }}
PODMAN_EXECUTABLE: ${{ steps.boundary.outputs.podman_executable }}
QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }}
RUNNER_CONTRACT: ${{ steps.gpu_resources.outputs.runner_contract }}
RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }}
RUNTIME_DIRECTORY_UNIT: ${{ steps.boundary.outputs.runtime_directory_unit }}
STORAGE_CONFIG: ${{ steps.boundary.outputs.storage_config }}
USER_MANAGER_UNIT: ${{ steps.boundary.outputs.user_manager_unit }}
shell: bash
run: |
set -euo pipefail
live_test="test/e2e/live/native-runtime-qualification-case.test.ts"
[[ -f "$CANDIDATE_DIRECTORY/$live_test" && ! -L "$CANDIDATE_DIRECTORY/$live_test" ]] || {
echo "::error::Candidate commit does not provide the native runtime qualification case executor" >&2
exit 1
}
receipt_directory="${RUNNER_TEMP}/native-runtime-case"
install -d -m 0700 "$receipt_directory"
sudo chown "$ACCOUNT_UID:$ACCOUNT_GID" "$receipt_directory"
cd "$CANDIDATE_DIRECTORY"
sudo -u "$ACCOUNT" env -i \
CI=true \
CONTAINERS_CONF="$CONTAINERS_CONFIG" \
CONTAINERS_STORAGE_CONF="$STORAGE_CONFIG" \
E2E_DEFAULT_ENABLED=0 \
E2E_JOB=1 \
HOME="$QUALIFICATION_HOME" \
LANG=C.UTF-8 \
NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RECEIPT="$receipt_directory/execution.json" \
NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_ROW="$QUALIFICATION_ROW" \
NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_PODMAN_EXECUTABLE="$PODMAN_EXECUTABLE" \
NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RUNNER_CONTRACT="$RUNNER_CONTRACT" \
NEMOCLAW_RUN_LIVE_E2E=1 \
PATH="$GUARD_DIRECTORY:$HELPER_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \
XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \
"$CANDIDATE_DIRECTORY/node_modules/.bin/vitest" run \
--config "$CANDIDATE_DIRECTORY/vitest.config.ts" \
--project e2e-live \
"$live_test"
sudo systemctl stop "$USER_MANAGER_UNIT" "$RUNTIME_DIRECTORY_UNIT"
! systemctl is-active --quiet "$USER_MANAGER_UNIT" && ! systemctl is-active --quiet "$RUNTIME_DIRECTORY_UNIT" || {
echo "::error::Qualification systemd user lifecycle remained active after candidate execution" >&2
exit 1
}
sudo pkill -KILL -u "$ACCOUNT_UID" 2>/dev/null || true
- name: Verify Docker stayed unavailable
shell: bash
run: |
set -euo pipefail
! systemctl is-active --quiet docker.service
! systemctl is-active --quiet docker.socket
! pgrep -x dockerd >/dev/null
[[ ! -S /var/run/docker.sock && ! -S /run/docker.sock ]]
- name: Validate receipts and emit bounded evidence
env:
EVIDENCE_DIRECTORY: ${{ runner.temp }}/native-runtime-evidence
EXECUTION_RECEIPT_PATH: ${{ runner.temp }}/native-runtime-case/execution.json
INSTALLER_RECEIPT_DIRECTORY: ${{ runner.temp }}/native-runtime-installer/receipts
NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }}
shell: bash
run: |
set -euo pipefail
sudo --preserve-env=EVIDENCE_DIRECTORY,EXECUTION_RECEIPT_PATH,INSTALLER_RECEIPT_DIRECTORY,QUALIFICATION_ROW \
"$NODE_DIRECTORY/node" --experimental-strip-types --no-warnings \
.qualification-workflow/tools/e2e/native-runtime-qualification-producer-evidence.mts
sudo chown -R "$(id -u):$(id -g)" "$EVIDENCE_DIRECTORY"
- name: Remove qualification resources
if: always()
env:
ACCOUNT: ${{ steps.boundary.outputs.account }}
ACCOUNT_CREATED: ${{ steps.boundary.outputs.account_created }}
shell: bash
run: |
set -euo pipefail
reported_account="${ACCOUNT:-}"
reported_created="${ACCOUNT_CREATED:-}"
ownership_marker="/run/nemoclaw-native-runtime-owner-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
account=""
uid=""
gid=""
if [[ -e "$ownership_marker" || -L "$ownership_marker" ]]; then
[[ -f "$ownership_marker" && ! -L "$ownership_marker" && "$(stat -c '%u:%g:%a' "$ownership_marker")" == "0:0:400" ]] || {
echo "::error::Qualification account ownership marker cleanup target is invalid" >&2
exit 1
}
ownership="$(sudo cat "$ownership_marker")"
[[ "$ownership" =~ ^nemoclawq:([0-9]+):([0-9]+)$ ]] || {
echo "::error::Qualification account ownership marker content is invalid" >&2
exit 1
}
account="nemoclawq"
uid="${BASH_REMATCH[1]}"
gid="${BASH_REMATCH[2]}"
[[ -z "$reported_account" || "$reported_account" == "$account" ]] || {
echo "::error::Qualification account output does not match its ownership marker" >&2
exit 1
}
if getent passwd "$uid" >/dev/null && ! getent passwd "$account" >/dev/null; then
echo "::error::Qualification account UID belongs to a different host account" >&2
exit 1
fi
if getent passwd "$account" >/dev/null; then
[[ "$(id -u "$account")" == "$uid" && "$(id -g "$account")" == "$gid" ]] || {
echo "::error::Qualification account identity changed before cleanup" >&2
exit 1
}
fi
if getent group "$account" >/dev/null; then
[[ "$(getent group "$account")" == "${account}:x:${gid}:" ]] || {
echo "::error::Qualification private group identity changed before cleanup" >&2
exit 1
}
elif getent passwd "$account" >/dev/null; then
echo "::error::Qualification private group disappeared before cleanup" >&2
exit 1
fi
runtime_dir="/run/user/${uid}"
runtime_directory_unit="user-runtime-dir@${uid}.service"
user_manager_unit="user@${uid}.service"
user_manager_dropin_directory="/run/systemd/system/${user_manager_unit}.d"
user_manager_dropin="${user_manager_dropin_directory}/50-nemoclaw-native-runtime.conf"
trusted_user_unit_path="/usr/lib/systemd/user:/lib/systemd/user"
storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
podman_executable="/nemoclaw-native-runtime-podman-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
apparmor_profile="${storage_config_directory}/podman.apparmor"
pasta_apparmor_profile="${storage_config_directory}/pasta.apparmor"
helper_directory="/nemoclaw-native-runtime-helpers-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
pasta_executable="${helper_directory}/pasta"
registry_auth_directory="${storage_config_directory}/registry-auth"
registry_auth_file="${registry_auth_directory}/auth.json"
runner_contract="${storage_config_directory}/runner-contract.json"
resource_directory="/var/tmp/nemoclaw-native-runtime-resources-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
model_directory="${resource_directory}/model"
sudo systemctl stop "$user_manager_unit" "$runtime_directory_unit" 2>/dev/null || true
! systemctl is-active --quiet "$user_manager_unit" && ! systemctl is-active --quiet "$runtime_directory_unit" || {
echo "::error::Qualification systemd user lifecycle remained active during cleanup" >&2
exit 1
}
if getent passwd "$account" >/dev/null; then
sudo pkill -KILL -u "$uid" 2>/dev/null || true
fi
if [[ -e "$user_manager_dropin_directory" || -L "$user_manager_dropin_directory" ]]; then
[[ -d "$user_manager_dropin_directory" && ! -L "$user_manager_dropin_directory" && "$(stat -c '%u:%g:%a' "$user_manager_dropin_directory")" == "0:0:755" ]] || {
echo "::error::Qualification systemd user-manager drop-in directory cleanup target is invalid" >&2
exit 1
}
[[ -f "$user_manager_dropin" && ! -L "$user_manager_dropin" && "$(stat -c '%u:%g:%a:%h' "$user_manager_dropin")" == "0:0:444:1" ]] || {
echo "::error::Qualification systemd user-manager drop-in cleanup target is invalid" >&2
exit 1
}
expected_user_manager_dropin="$(printf '[Service]\nEnvironment="SYSTEMD_UNIT_PATH=%s"' "$trusted_user_unit_path")"
[[ "$(cat "$user_manager_dropin")" == "$expected_user_manager_dropin" ]] || {
echo "::error::Qualification systemd user-manager drop-in content changed before cleanup" >&2
exit 1
}
sudo unlink "$user_manager_dropin"
sudo rmdir "$user_manager_dropin_directory"
sudo systemctl daemon-reload
fi
if [[ -e "$storage_config_directory" || -L "$storage_config_directory" ]]; then
[[ -d "$storage_config_directory" && ! -L "$storage_config_directory" ]] || {
echo "::error::Qualification storage configuration cleanup target is invalid" >&2
exit 1
}
if [[ -e "$apparmor_profile" || -L "$apparmor_profile" ]]; then
[[ -f "$apparmor_profile" && ! -L "$apparmor_profile" && "$(stat -c '%u:%g:%a' "$apparmor_profile")" == "0:0:444" ]] || {
echo "::error::Qualification AppArmor profile cleanup target is invalid" >&2
exit 1
}
sudo apparmor_parser -R "$apparmor_profile"
sudo rm -f -- "$apparmor_profile"
fi
if [[ -e "$pasta_apparmor_profile" || -L "$pasta_apparmor_profile" ]]; then
[[ -f "$pasta_apparmor_profile" && ! -L "$pasta_apparmor_profile" && "$(stat -c '%u:%g:%a' "$pasta_apparmor_profile")" == "0:0:444" ]] || {
echo "::error::Qualification pasta AppArmor profile cleanup target is invalid" >&2
exit 1
}
sudo apparmor_parser -R "$pasta_apparmor_profile"
sudo rm -f -- "$pasta_apparmor_profile"
fi
if [[ -e "$registry_auth_directory" || -L "$registry_auth_directory" ]]; then
[[ -d "$registry_auth_directory" && ! -L "$registry_auth_directory" && "$(stat -c '%u:%g:%a' "$registry_auth_directory")" == "${uid}:${gid}:700" ]] || {
echo "::error::Qualification registry authentication directory cleanup target is invalid" >&2
exit 1
}
if sudo test -e "$registry_auth_file" || sudo test -L "$registry_auth_file"; then
sudo test -f "$registry_auth_file" &&
sudo test ! -L "$registry_auth_file" &&
[[ "$(sudo stat -c '%u:%g:%a:%h' -- "$registry_auth_file")" == "${uid}:${gid}:600:1" ]] || {
echo "::error::Qualification registry authentication file cleanup target is invalid" >&2
exit 1
}
sudo unlink "$registry_auth_file"
fi
sudo rmdir "$registry_auth_directory"
fi
if [[ -e "$runner_contract" || -L "$runner_contract" ]]; then
[[ -f "$runner_contract" && ! -L "$runner_contract" && "$(stat -c '%u:%g:%a' "$runner_contract")" == "0:0:444" ]] || {
echo "::error::Qualification GPU runner contract cleanup target is invalid" >&2
exit 1
}
sudo unlink "$runner_contract"
fi
if [[ -e "$storage_config_directory/containers.conf" || -L "$storage_config_directory/containers.conf" ]]; then
[[ -f "$storage_config_directory/containers.conf" && ! -L "$storage_config_directory/containers.conf" && "$(stat -c '%u:%g:%a' "$storage_config_directory/containers.conf")" == "0:0:444" ]] || {
echo "::error::Qualification containers configuration cleanup target is invalid" >&2
exit 1
}
sudo unlink "$storage_config_directory/containers.conf"
fi
sudo rm -f -- "$storage_config_directory/storage.conf"
sudo rmdir "$storage_config_directory"
fi
if [[ -e "$resource_directory" || -L "$resource_directory" ]]; then
[[ -d "$resource_directory" && ! -L "$resource_directory" && ("$(stat -c '%u:%g:%a' "$resource_directory")" == "0:0:711" || "$(stat -c '%u:%g:%a' "$resource_directory")" == "0:0:555") ]] || {
echo "::error::Qualification GPU resource directory cleanup target is invalid" >&2
exit 1
}
if sudo test -e "$model_directory" || sudo test -L "$model_directory"; then
model_mode="$(sudo stat -c '%u:%g:%a' -- "$model_directory")"
sudo test -d "$model_directory" &&
sudo test ! -L "$model_directory" &&
[[ "$model_mode" == "${uid}:${gid}:700" || "$model_mode" == "0:0:555" ]] || {
echo "::error::Qualification GPU model directory cleanup target is invalid" >&2
exit 1
}
for file in config.json generation_config.json merges.txt model.safetensors tokenizer.json tokenizer_config.json vocab.json; do
target="${model_directory}/${file}"
if sudo test -e "$target" || sudo test -L "$target"; then
file_mode="$(sudo stat -c '%u:%g:%a:%h' -- "$target")"
sudo test -f "$target" &&
sudo test ! -L "$target" &&
[[ "$file_mode" == "${uid}:${gid}:600:1" || "$file_mode" == "0:0:444:1" ]] || {
echo "::error::Qualification GPU model file cleanup target is invalid: $file" >&2
exit 1
}
sudo unlink "$target"
fi
done
sudo rmdir "$model_directory" || {
echo "::error::Qualification GPU model directory contains unexpected entries" >&2
exit 1
}
fi
sudo rmdir "$resource_directory" || {
echo "::error::Qualification GPU resource directory contains unexpected entries" >&2
exit 1
}
fi
if [[ -e "$podman_executable" || -L "$podman_executable" ]]; then
[[ -f "$podman_executable" && ! -L "$podman_executable" && "$(stat -c '%u:%g:%a' "$podman_executable")" == "0:0:555" ]] || {
echo "::error::Qualification Podman executable cleanup target is invalid" >&2
exit 1
}
sudo rm -f -- "$podman_executable"
fi
if [[ -e "$helper_directory" || -L "$helper_directory" ]]; then
[[ -d "$helper_directory" && ! -L "$helper_directory" && "$(stat -c '%u:%g:%a' "$helper_directory")" == "0:0:555" ]] || {
echo "::error::Qualification helper directory cleanup target is invalid" >&2
exit 1
}
[[ -f "$pasta_executable" && ! -L "$pasta_executable" && "$(stat -c '%u:%g:%a' "$pasta_executable")" == "0:0:555" ]] || {
echo "::error::Qualification pasta executable cleanup target is invalid" >&2
exit 1
}
sudo rm -f -- "$pasta_executable"
sudo rmdir "$helper_directory"
fi
if [[ -e "$runtime_dir" || -L "$runtime_dir" ]]; then
echo "::error::Qualification runtime directory remains after its systemd cleanup" >&2
exit 1
fi
if getent passwd "$account" >/dev/null; then
sudo userdel --remove "$account"
fi
if getent group "$account" >/dev/null; then
sudo groupdel "$account"
fi
if getent passwd "$account" >/dev/null; then
echo "::error::Qualification account still exists after cleanup" >&2
exit 1
fi
if getent group "$account" >/dev/null; then
echo "::error::Qualification private group still exists after cleanup" >&2
exit 1
fi
if grep -q "^${account}:" /etc/subuid /etc/subgid; then
echo "::error::Qualification subordinate-ID authorization remains after cleanup" >&2
exit 1
fi
[[ ! -e "/run/user/${uid}" && ! -L "/run/user/${uid}" ]] || {
echo "::error::Qualification runtime directory remains after cleanup" >&2
exit 1
}
storage_config_directory="/run/nemoclaw-native-runtime-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${uid}"
[[ ! -e "$storage_config_directory" && ! -L "$storage_config_directory" ]] || {
echo "::error::Qualification storage configuration remains after cleanup" >&2
exit 1
}
[[ ! -e "$podman_executable" && ! -L "$podman_executable" ]] || {
echo "::error::Qualification Podman executable remains after cleanup" >&2
exit 1
}
[[ ! -e "$helper_directory" && ! -L "$helper_directory" ]] || {
echo "::error::Qualification helper directory remains after cleanup" >&2
exit 1
}
[[ ! -e "$resource_directory" && ! -L "$resource_directory" ]] || {
echo "::error::Qualification GPU resource directory remains after cleanup" >&2
exit 1
}
sudo rm -f -- "$ownership_marker"
elif [[ -n "$reported_account" || "$reported_created" == "true" ]]; then
echo "::error::Qualification account output exists without its ownership marker" >&2
exit 1
fi
[[ ! -e "$ownership_marker" && ! -L "$ownership_marker" ]] || {
echo "::error::Qualification account ownership marker remains after cleanup" >&2
exit 1
}
- name: Upload the qualification case evidence
if: success()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: ${{ matrix.artifactName }}
path: ${{ runner.temp }}/native-runtime-evidence/
native-runtime-qualification-producer-aggregate:
name: Aggregate native runtime qualification evidence
needs:
[
generate-matrix,
native-runtime-qualification-producer-plan,
native-runtime-qualification-producer,
]
if: ${{ always() && needs.native-runtime-qualification-producer-plan.result == 'success' && needs.native-runtime-qualification-producer.result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'native-runtime-qualification-producer') }}
runs-on: ubuntu-24.04
timeout-minutes: 15
permissions:
actions: read
contents: read
pull-requests: read
steps:
- name: Check out the qualification aggregator
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ github.repository }}
ref: ${{ github.workflow_sha }}
path: .qualification-aggregate
persist-credentials: false
sparse-checkout: |
src/lib/onboard/runtime-provider/native-qualification-authority.ts
test/e2e/registry/native-runtime-qualification.ts
tools/e2e/native-runtime-qualification-producer-plan.mts
tools/e2e/native-runtime-qualification-producer-aggregate.mts
sparse-checkout-cone-mode: false
- name: Download the exact case evidence cohort
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
pattern: native-runtime-qualification-evidence-${{ inputs.checkout_sha }}-*
path: ${{ runner.temp }}/native-runtime-case-artifacts
merge-multiple: false
- name: Resolve this aggregate job identity
id: aggregate-job
env:
GH_TOKEN: ${{ github.token }}
PRODUCER_RUN_ATTEMPT: ${{ github.run_attempt }}
PRODUCER_RUN_ID: ${{ github.run_id }}
shell: bash
run: |
set -euo pipefail
jobs="$(gh api --method GET \
"repos/${GITHUB_REPOSITORY}/actions/runs/${PRODUCER_RUN_ID}/attempts/${PRODUCER_RUN_ATTEMPT}/jobs" \
-f per_page=100)"
total_count="$(jq -er '.total_count | select(type == "number" and . >= 1)' <<<"$jobs")" || {
echo "::error::Aggregate job lookup returned an invalid job count" >&2
exit 1
}
(( total_count <= 100 )) || {
echo "::error::Aggregate job lookup exceeds the bounded 100-job page" >&2
exit 1
}
job_id="$(jq -er \
--arg name 'Aggregate native runtime qualification evidence' \
--argjson runId "$PRODUCER_RUN_ID" \
--argjson attempt "$PRODUCER_RUN_ATTEMPT" '
[.jobs[] | select(
.name == $name and
.run_id == $runId and
.run_attempt == $attempt and
.status == "in_progress"
)] |
select(length == 1) |
.[0].id
' <<<"$jobs")" || {
echo "::error::Could not resolve one in-progress aggregate job identity" >&2
exit 1
}
[[ "$job_id" =~ ^[1-9][0-9]{0,19}$ ]] || {
echo "::error::Aggregate job identity is invalid" >&2
exit 1
}
printf 'job_id=%s\n' "$job_id" >>"$GITHUB_OUTPUT"
- name: Set up Node for qualification aggregation
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22.19.0
- name: Validate and aggregate all 24 case receipts
working-directory: .qualification-aggregate
env:
AGGREGATE_JOB_ID: ${{ steps.aggregate-job.outputs.job_id }}
CASE_ARTIFACT_ROOT: ${{ runner.temp }}/native-runtime-case-artifacts
EVIDENCE_DIRECTORY: ${{ runner.temp }}/native-runtime-aggregate
QUALIFICATION_PLAN: ${{ needs.native-runtime-qualification-producer-plan.outputs.matrix }}
run: >-
node --experimental-strip-types --no-warnings
tools/e2e/native-runtime-qualification-producer-aggregate.mts
- name: Upload aggregate evidence
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: native-runtime-qualification-${{ inputs.checkout_sha }}
path: ${{ runner.temp }}/native-runtime-aggregate/
if-no-files-found: error
retention-days: 30
compression-level: 9
retired-selector-compatibility:
needs: generate-matrix
if: ${{ inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',credential-migration,') || contains(format(',{0},', inputs.jobs), ',credential-sanitization,') || contains(format(',{0},', inputs.jobs), ',diagnostics,') || contains(format(',{0},', inputs.jobs), ',docs-validation,') || contains(format(',{0},', inputs.jobs), ',gateway-drift-preflight,') || contains(format(',{0},', inputs.jobs), ',gateway-health-honest,') || contains(format(',{0},', inputs.jobs), ',onboard-negative-paths,') || contains(format(',{0},', inputs.jobs), ',openshell-version-pin,') || contains(format(',{0},', inputs.jobs), ',sandbox-rebuild,') || contains(format(',{0},', inputs.jobs), ',ubuntu-repo-cli-smoke,') || contains(format(',{0},', inputs.jobs), ',upgrade-stale-sandbox,') || contains(format(',{0},', inputs.targets), ',sandbox-rebuild,') || contains(format(',{0},', inputs.targets), ',upgrade-stale-sandbox,')) }}
runs-on: ubuntu-latest
timeout-minutes: 15
env:
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/retired-selector-compatibility
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@c246409193a31133cab10c8a3589001cc0d59eb3
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Verify retired selector replacements
env:
JOBS: ${{ inputs.jobs }}
TARGETS: ${{ inputs.targets }}
run: npx tsx tools/e2e/retired-selector-compatibility.mts
- name: Upload retired selector compatibility evidence
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-retired-selector-compatibility
path: e2e-artifacts/live/retired-selector-compatibility/
staging-brev-launchable:
name: Exact staging Brev Launchable
needs: generate-matrix
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && ((inputs.jobs == 'staging-brev-launchable' && inputs.targets == '') || (inputs.include_staging_brev_launchable && inputs.jobs == '' && inputs.targets == '')) }}
runs-on: ubuntu-latest
timeout-minutes: 180
permissions:
contents: read
concurrency:
group: staging-brev-launchable-cpu
queue: max
cancel-in-progress: false
env:
CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }}
E2E_JOB: "1"
INSTANCE_NAME: nclaw-e2e-${{ github.run_id }}-${{ github.run_attempt }}
steps:
- name: Checkout trusted Launchable lane
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.workflow_sha }}
persist-credentials: false
sparse-checkout: |
tools/e2e/brev-launchable-e2e.sh
sparse-checkout-cone-mode: false
- id: workspace
name: Prepare the trusted lane
env:
BREV_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.BREV_API_KEY || '' }}
BREV_CLI_SHA256: d4aa49db1716f10308a6587778a676a0c0076bd48a212d86a421ab9550bc8f32
BREV_CLI_VERSION: 0.6.334
BREV_ORG_ID: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.BREV_ORG_ID || '' }}
run: |
set -euo pipefail
work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-launchable-e2e.XXXXXX")"
chmod 700 "$work_dir"
archive="${RUNNER_TEMP}/brev-cli.tar.gz"
curl -fsSL -o "$archive" "https://github.com/brevdev/brev-cli/releases/download/v${BREV_CLI_VERSION}/brev-cli_${BREV_CLI_VERSION}_linux_amd64.tar.gz"
printf '%s %s\n' "$BREV_CLI_SHA256" "$archive" | sha256sum -c -
tar -xzf "$archive" -C "${RUNNER_TEMP}" brev && sudo install -m 0755 "${RUNNER_TEMP}/brev" /usr/local/bin/brev
brev login --api-key "$BREV_API_KEY" --org-id "$BREV_ORG_ID"
printf 'work_dir=%s\n' "$work_dir" >> "$GITHUB_OUTPUT"
- name: Build, deploy, verify, test, and clean up
env:
BREV_LAUNCHABLE_ID: ${{ vars.NEMOCLAW_STAGING_LAUNCHABLE_ID }}
GH_TOKEN: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.NEMOCLAW_IMAGE_DISPATCH_TOKEN || '' }}
NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && secrets.NVIDIA_INFERENCE_API_KEY || '' }}
WORK_DIR: ${{ steps.workspace.outputs.work_dir }}
run: tools/e2e/brev-launchable-e2e.sh
- name: Upload Launchable evidence
if: ${{ always() && steps.workspace.outputs.work_dir != '' }}
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: staging-brev-launchable-${{ env.CANDIDATE_SHA }}-${{ github.run_id }}-${{ github.run_attempt }}
path: |
${{ steps.workspace.outputs.work_dir }}/lane.log
${{ steps.workspace.outputs.work_dir }}/launchable-e2e.json
${{ steps.workspace.outputs.work_dir }}/full-e2e.log
${{ steps.workspace.outputs.work_dir }}/cleanup.json
live:
needs: [base-image-publication, generate-matrix]
if: ${{ needs.generate-matrix.outputs.matrix != '[]' }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 45
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }}
env:
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live
NEMOCLAW_LANGCHAIN_DEEPAGENTS_CODE_SANDBOX_BASE_IMAGE_REF: ${{ needs.base-image-publication.outputs.dcode_base_ref }}
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false
# Keep only the credential-bearing step anchored. Cleanup mappings stay
# explicit because strict YAML decoders reject 100 or more aliases here.
- &dockerhub-auth
name: Authenticate to Docker Hub
uses: NVIDIA/NemoClaw/.github/actions/docker-auth-setup@05fa6b810017752ab21148cb7e9d82d12a88c92f
with:
auth-required: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && '1' || '0' }}
username: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && secrets.DOCKERHUB_USERNAME || '' }}
token: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && inputs.checkout_sha == '' && secrets.DOCKERHUB_TOKEN || '' }}
- name: Configure live E2E trace directory
env:
TARGET_ID: ${{ matrix.id }}
shell: bash
run: |
set -euo pipefail
printf 'NEMOCLAW_TRACE_DIR=%s\n' "${RUNNER_TEMP}/nemoclaw-e2e-traces/${TARGET_ID}" >> "${GITHUB_ENV}"
# invalidState: the selected PR-modifiable TUI check needs a PTY driver,
# but the fixed GitHub-hosted runner image does not provide expect.
# sourceBoundary: privileged host setup runs from the first-party
# host-dependency-setup action pinned to an immutable full SHA, never the
# PR-controlled target ref; the check only consumes expect without privilege.
# whyNotSourceFix: GitHub-hosted jobs cannot use a repository-owned host
# image, and caching privileged dpkg state between clean runners is not
# supported.
# regressionTest: the workflow-boundary suite pins this target, condition,
# ordering, action provenance, and package mapping.
# removalCondition: remove the install when the hosted runner supplies
# expect or the acceptance check no longer requires a PTY.
- name: Install Deep Agents Code TUI host dependencies
if: ${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' }}
uses: NVIDIA/NemoClaw/.github/actions/host-dependency-setup@4def1501b34ce586f83b91af50a66b5d22b31d75
with:
packages: expect
# Configure NEMOCLAW_TRACE_DIR before workspace prep so every child
# command writes raw traces under runner temp, never under upload roots.
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Record immutable Deep Agents Code base evidence
if: ${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' }}
env:
BASE_CONTRACT: ${{ needs.base-image-publication.outputs.dcode_base_contract }}
CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }}
TARGET_ID: ${{ matrix.id }}
shell: bash
run: |
set -euo pipefail
evidence_dir="${E2E_ARTIFACT_DIR}/${TARGET_ID}"
install -d -m 0700 "${evidence_dir}"
jq -e -n \
--arg candidateSha "${CANDIDATE_SHA}" \
--argjson baseContract "${BASE_CONTRACT}" \
'{contractVersion: 1, candidateSha: $candidateSha, base: $baseContract}
| select(
(.candidateSha | test("^[0-9a-f]{40}$"))
and (.base.reference | test("@sha256:[0-9a-f]{64}$"))
)' >"${evidence_dir}/dcode-base-image.json"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@c246409193a31133cab10c8a3589001cc0d59eb3
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
# invalidState: a profile plugin installed with --no-deps can import even
# when an incomplete base image omitted its required upstream packages.
# sourceBoundary: this trusted workflow scopes the repo-owned stripped-base
# build to the exact DCode target; the production Dockerfile must reject it
# at the isolated import gate before its later dependency-consistency check.
# whyNotSourceFix: dependency completeness belongs to the hash-locked base;
# resolving dependencies during local plugin install would duplicate that
# trust boundary, so the regression constructs the invalid input instead.
# regressionTest: workflow-boundary tests pin the target, script, and
# ordering; build-gate tests pin the base build and failure contract.
# removalCondition: remove only if package installation no longer uses
# --no-deps or an equivalent earlier build gate proves both imports.
- name: Verify DCode profile import gate rejects missing base dependencies
if: ${{ matrix.id == 'ubuntu-repo-cloud-langchain-deepagents-code' }}
shell: bash
run: bash scripts/check-dcode-profile-import-gate.sh
- name: Run live E2E tests
env:
E2E_TARGET_ID: ${{ matrix.id }}
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
TARGET_ID: ${{ matrix.id }}
run: |
set -euo pipefail
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/registry-targets.test.ts --selector "^${TARGET_ID}$"
# The sanitizer reads raw traces only after checking the workflow-owned
# runner-temp path, then writes the timing-only file into upload roots.
- name: Build trusted live E2E timing summary
if: always()
env:
TARGET_ID: ${{ matrix.id }}
shell: bash
run: |
set -euo pipefail
expected_trace_dir="${RUNNER_TEMP}/nemoclaw-e2e-traces/${TARGET_ID}"
if [ -z "${RUNNER_TEMP}" ] || [ "${NEMOCLAW_TRACE_DIR}" != "${expected_trace_dir}" ]; then
echo "::error::Refusing to sanitize unexpected raw trace path" >&2
exit 1
fi
python3 scripts/e2e/sanitize-trace-timing.py \
"${NEMOCLAW_TRACE_DIR}" \
"${E2E_ARTIFACT_DIR}/${TARGET_ID}"
# Cleanup intentionally runs after sanitization and before upload so raw
# trace JSON never becomes part of the uploaded artifact surface.
- name: Delete raw live E2E traces
if: always()
env:
TARGET_ID: ${{ matrix.id }}
shell: bash
run: |
set -euo pipefail
expected_trace_dir="${RUNNER_TEMP}/nemoclaw-e2e-traces/${TARGET_ID}"
if [ -z "${RUNNER_TEMP}" ] || [ "${NEMOCLAW_TRACE_DIR}" != "${expected_trace_dir}" ]; then
echo "::error::Refusing to delete unexpected raw trace path" >&2
exit 1
fi
rm -rf -- "${NEMOCLAW_TRACE_DIR}"
- name: Summarize artifacts
if: always()
env:
TARGET_ID: ${{ matrix.id }}
TARGET_LABEL: ${{ matrix.label }}
run: |
python3 - <<'PY' >> "$GITHUB_STEP_SUMMARY"
import json
import os
from pathlib import Path
root = Path(os.environ["E2E_ARTIFACT_DIR"]) / os.environ["TARGET_ID"]
plan_path = root / "run-plan.json"
print("## E2E Targets")
print()
print("- Project: `e2e-live`")
print(f"- Target: `{os.environ['TARGET_ID']}`")
print(f"- Label: `{os.environ['TARGET_LABEL']}`")
print(f"- Artifact root: `{root}`")
print()
print("| Target | Manifest | Expected state | Suites | Phases |")
print("| --- | --- | --- | --- | --- |")
if plan_path.exists():
plan = json.loads(plan_path.read_text(encoding="utf-8"))
suites = ", ".join(plan.get("suiteIds") or []) or "(none)"
phases = ", ".join(plan.get("phases") or []) or "(none)"
print(
"| "
f"`{plan.get('targetId') or os.environ['TARGET_ID']}` | "
f"`{plan.get('manifestPath') or 'not-yet-defined'}` | "
f"`{plan.get('expectedStateId') or 'not-yet-defined'}` | "
f"{suites} | {phases} |"
)
else:
print(
"| "
f"`{os.environ['TARGET_ID']}` | `(missing run-plan.json)` | "
"`(missing run-plan.json)` | `(missing)` | `(missing)` |"
)
PY
- name: Upload E2E artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-${{ matrix.id }}
path: |
e2e-artifacts/live/${{ matrix.id }}/run-plan.json
e2e-artifacts/live/${{ matrix.id }}/target.json
e2e-artifacts/live/${{ matrix.id }}/target-result.json
e2e-artifacts/live/${{ matrix.id }}/test-progress.json
e2e-artifacts/live/${{ matrix.id }}/environment.result.json
e2e-artifacts/live/${{ matrix.id }}/onboarding.result.json
e2e-artifacts/live/${{ matrix.id }}/state-validation.result.json
e2e-artifacts/live/${{ matrix.id }}/dcode-base-image.json
e2e-artifacts/live/${{ matrix.id }}/cloud-onboard-trace-timing-summary.json
e2e-artifacts/live/risk-signal.json
e2e-artifacts/live/${{ matrix.id }}/actions/
e2e-artifacts/live/${{ matrix.id }}/logs/
e2e-artifacts/live/${{ matrix.id }}/shell/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
# Credential-free tests opt in with a tag beside the test. Discovery supplies
# only a validated test ID, file, and Vitest project; this E2E workflow owns
# the shared job's runner, setup, timeout, permissions, and artifact policy.
shared-e2e:
name: Shared E2E (${{ matrix.id }})
needs: generate-matrix
if: ${{ needs.generate-matrix.outputs.test_matrix != '[]' }}
runs-on: ubuntu-latest
timeout-minutes: 15
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.test_matrix) }}
env:
CHECK_DOC_LINKS_REMOTE: "0"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/${{ matrix.id }}
E2E_TARGET_ID: ${{ matrix.id }}
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_RUN_LIVE_E2E: "1"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@c246409193a31133cab10c8a3589001cc0d59eb3
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Run tagged credential-free test
env:
TEST_FILE: ${{ matrix.file }}
TEST_PROJECT: ${{ matrix.project }}
run: |
set -euo pipefail
npx vitest run --project "${TEST_PROJECT}" "${TEST_FILE}" \
--tags-filter=e2e/credential-free \
--silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts
- name: Upload test artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
catalogue-standard:
name: ${{ matrix.display_name }}
needs: generate-matrix
if: ${{ needs.generate-matrix.outputs.catalogue_standard_matrix != '[]' }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.catalogue_standard_matrix) }}
uses: ./.github/workflows/e2e-standard-profile.yaml
with:
candidate_repository: ${{ inputs.checkout_repository || github.repository }}
candidate_sha: ${{ inputs.checkout_sha || github.sha }}
risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }}
risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }}
cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
credential_boundary: no provider credential
target_id: ${{ matrix.target_id }}
catalogue_id: ${{ matrix.id }}
runner: ${{ matrix.runner_key != '' && fromJSON(needs.generate-matrix.outputs.runner_routing)[matrix.runner_key] || matrix.runner }}
checkout_sha: ${{ inputs.checkout_sha }}
workflow_sha: ${{ inputs.workflow_sha }}
test_file: ${{ matrix.test_file }}
timeout_minutes: ${{ matrix.timeout_minutes }}
install_mode: ${{ matrix.install_mode }}
install_non_interactive: ${{ matrix.install_non_interactive }}
restore_cli: ${{ matrix.restore_cli }}
cloudflared: ${{ matrix.cloudflared }}
host_packages: ${{ matrix.host_packages }}
host_preparation: ${{ matrix.host_preparation }}
runner_comparison: ${{ matrix.runner_comparison }}
compatible_api_key: ${{ matrix.compatible_api_key }}
github_token: false
shard: ${{ matrix.shard }}
artifact_layout: ${{ matrix.artifact_layout }}
trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }}
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
catalogue-nvidia-api:
name: ${{ matrix.display_name }}
needs: generate-matrix
if: ${{ needs.generate-matrix.outputs.catalogue_nvidia_api_matrix != '[]' }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.catalogue_nvidia_api_matrix) }}
uses: ./.github/workflows/e2e-standard-profile.yaml
with:
candidate_repository: ${{ inputs.checkout_repository || github.repository }}
candidate_sha: ${{ inputs.checkout_sha || github.sha }}
risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }}
risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }}
cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
credential_boundary: NVIDIA API key
target_id: ${{ matrix.target_id }}
catalogue_id: ${{ matrix.id }}
runner: ${{ matrix.runner_key != '' && fromJSON(needs.generate-matrix.outputs.runner_routing)[matrix.runner_key] || matrix.runner }}
checkout_sha: ${{ inputs.checkout_sha }}
workflow_sha: ${{ inputs.workflow_sha }}
test_file: ${{ matrix.test_file }}
timeout_minutes: ${{ matrix.timeout_minutes }}
install_mode: ${{ matrix.install_mode }}
install_non_interactive: ${{ matrix.install_non_interactive }}
restore_cli: ${{ matrix.restore_cli }}
cloudflared: ${{ matrix.cloudflared }}
host_packages: ${{ matrix.host_packages }}
host_preparation: ${{ matrix.host_preparation }}
runner_comparison: ${{ matrix.runner_comparison }}
compatible_api_key: ${{ matrix.compatible_api_key }}
github_token: false
shard: ${{ matrix.shard }}
artifact_layout: ${{ matrix.artifact_layout }}
trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }}
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
catalogue-nvidia-inference:
name: ${{ matrix.display_name }}
needs: generate-matrix
if: ${{ needs.generate-matrix.outputs.catalogue_nvidia_inference_matrix != '[]' }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.catalogue_nvidia_inference_matrix) }}
uses: ./.github/workflows/e2e-standard-profile.yaml
with:
candidate_repository: ${{ inputs.checkout_repository || github.repository }}
candidate_sha: ${{ inputs.checkout_sha || github.sha }}
risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }}
risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }}
cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
credential_boundary: NVIDIA inference API key
target_id: ${{ matrix.target_id }}
catalogue_id: ${{ matrix.id }}
runner: ${{ matrix.runner_key != '' && fromJSON(needs.generate-matrix.outputs.runner_routing)[matrix.runner_key] || matrix.runner }}
checkout_sha: ${{ inputs.checkout_sha }}
workflow_sha: ${{ inputs.workflow_sha }}
test_file: ${{ matrix.test_file }}
timeout_minutes: ${{ matrix.timeout_minutes }}
install_mode: ${{ matrix.install_mode }}
install_non_interactive: ${{ matrix.install_non_interactive }}
restore_cli: ${{ matrix.restore_cli }}
cloudflared: ${{ matrix.cloudflared }}
host_packages: ${{ matrix.host_packages }}
host_preparation: ${{ matrix.host_preparation }}
runner_comparison: ${{ matrix.runner_comparison }}
compatible_api_key: ${{ matrix.compatible_api_key }}
github_token: false
shard: ${{ matrix.shard }}
artifact_layout: ${{ matrix.artifact_layout }}
trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }}
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
catalogue-github-read:
name: ${{ matrix.display_name }}
needs: generate-matrix
if: ${{ needs.generate-matrix.outputs.catalogue_github_read_matrix != '[]' }}
strategy:
fail-fast: false
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.catalogue_github_read_matrix) }}
uses: ./.github/workflows/e2e-standard-profile.yaml
with:
candidate_repository: ${{ inputs.checkout_repository || github.repository }}
candidate_sha: ${{ inputs.checkout_sha || github.sha }}
risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }}
risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }}
cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
credential_boundary: GitHub read token
target_id: ${{ matrix.target_id }}
catalogue_id: ${{ matrix.id }}
runner: ${{ matrix.runner_key != '' && fromJSON(needs.generate-matrix.outputs.runner_routing)[matrix.runner_key] || matrix.runner }}
checkout_sha: ${{ inputs.checkout_sha }}
workflow_sha: ${{ inputs.workflow_sha }}
test_file: ${{ matrix.test_file }}
timeout_minutes: ${{ matrix.timeout_minutes }}
install_mode: ${{ matrix.install_mode }}
install_non_interactive: ${{ matrix.install_non_interactive }}
restore_cli: ${{ matrix.restore_cli }}
cloudflared: ${{ matrix.cloudflared }}
host_packages: ${{ matrix.host_packages }}
host_preparation: ${{ matrix.host_preparation }}
runner_comparison: ${{ matrix.runner_comparison }}
compatible_api_key: ${{ matrix.compatible_api_key }}
github_token: true
shard: ${{ matrix.shard }}
artifact_layout: ${{ matrix.artifact_layout }}
trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }}
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
catalogue-brave-nvidia-inference:
name: ${{ matrix.display_name }}
needs: generate-matrix
if: ${{ needs.generate-matrix.outputs.catalogue_brave_nvidia_inference_matrix != '[]' }}
strategy:
fail-fast: false
max-parallel: 2
matrix:
include: ${{ fromJSON(needs.generate-matrix.outputs.catalogue_brave_nvidia_inference_matrix) }}
uses: ./.github/workflows/e2e-standard-profile.yaml
with:
candidate_repository: ${{ inputs.checkout_repository || github.repository }}
candidate_sha: ${{ inputs.checkout_sha || github.sha }}
risk_signal_expected_sha: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.checkout_sha || '' }}
risk_signal_correlation_id: ${{ github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && inputs.correlation_id || '' }}
cli_artifact_provenance: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
credential_boundary: Brave and NVIDIA inference API keys
target_id: ${{ matrix.target_id }}
catalogue_id: ${{ matrix.id }}
runner: ${{ matrix.runner_key != '' && fromJSON(needs.generate-matrix.outputs.runner_routing)[matrix.runner_key] || matrix.runner }}
checkout_sha: ${{ inputs.checkout_sha }}
workflow_sha: ${{ inputs.workflow_sha }}
test_file: ${{ matrix.test_file }}
timeout_minutes: ${{ matrix.timeout_minutes }}
install_mode: ${{ matrix.install_mode }}
install_non_interactive: ${{ matrix.install_non_interactive }}
restore_cli: ${{ matrix.restore_cli }}
cloudflared: ${{ matrix.cloudflared }}
host_packages: ${{ matrix.host_packages }}
host_preparation: ${{ matrix.host_preparation }}
runner_comparison: ${{ matrix.runner_comparison }}
compatible_api_key: ${{ matrix.compatible_api_key }}
github_token: false
shard: ${{ matrix.shard }}
artifact_layout: ${{ matrix.artifact_layout }}
trusted_main: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (inputs.checkout_sha == '' || needs.generate-matrix.outputs.e2e_credentials_allowed == 'true') }}
secrets:
DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
BRAVE_API_KEY: ${{ secrets.BRAVE_API_KEY }}
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
openshell-gateway-auth-contract:
needs: generate-matrix
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'openshell-gateway-auth-contract') }}
runs-on: ubuntu-latest
timeout-minutes: 20
env:
E2E_JOB: "1"
E2E_TARGET_ID: "openshell-gateway-auth-contract"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-gateway-auth-contract
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_OPENSHELL_PIN_VERSION: "0.0.101"
DOCKER_GRPC_PROBE_IMAGE: "node:22-trixie-slim@sha256:db8a96a63e5264607ada2d206758876ebbed6a12be2ada7517793cbfb0c2a29c"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false
- *dockerhub-auth
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@c246409193a31133cab10c8a3589001cc0d59eb3
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Install OpenShell CLI
run: env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh
- name: Pre-pull pinned gateway auth probe image
run: docker pull "$DOCKER_GRPC_PROBE_IMAGE"
- name: Run OpenShell gateway auth contract live test
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
if command -v openshell-gateway >/dev/null 2>&1; then
OPENSHELL_GATEWAY_BIN="$(command -v openshell-gateway)"
elif [ -x "$HOME/.local/bin/openshell-gateway" ]; then
OPENSHELL_GATEWAY_BIN="$HOME/.local/bin/openshell-gateway"
else
echo "::error::OpenShell gateway binary not found after install"
ls -la /usr/local/bin/openshell-gateway "$HOME/.local/bin/openshell-gateway" 2>&1 || true
exit 1
fi
export OPENSHELL_GATEWAY_BIN
echo "Using OPENSHELL_GATEWAY_BIN=$OPENSHELL_GATEWAY_BIN"
"$OPENSHELL_GATEWAY_BIN" --version
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/openshell-gateway-auth-source-contract.test.ts
- id: artifact_safety
name: Validate final OpenShell gateway auth contract artifacts
if: always()
run: node --experimental-strip-types --no-warnings tools/e2e/openshell-gateway-auth-artifact-safety.mts "$E2E_ARTIFACT_DIR"
- name: Upload OpenShell gateway auth contract artifacts
if: ${{ always() && steps.artifact_safety.outcome == 'success' && steps.artifact_safety.outputs.approved_path != '' }}
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-openshell-gateway-auth-contract
path: ${{ steps.artifact_safety.outputs.approved_path }}
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
mcp-bridge:
needs: generate-matrix
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'mcp-bridge') }}
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)[format('mcp-bridge-{0}', matrix.agent)] }}
permissions:
contents: read
# Keep each destructive agent lifecycle on a fresh runner. This bounds the
# blast radius of runner loss and gives every adapter independent evidence.
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
agent: [openclaw, hermes, deepagents]
env:
E2E_JOB: "1"
E2E_TARGET_ID: "mcp-bridge"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge/${{ matrix.agent }}
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_E2E_SHARD: ${{ matrix.agent }}
NEMOCLAW_MCP_BRIDGE_AGENT: ${{ matrix.agent }}
NEMOCLAW_OPENSHELL_CHANNEL: stable
NEMOCLAW_OPENSHELL_EXACT_MAIN_PROOF: "1"
NEMOCLAW_RUN_LIVE_E2E: "1"
OPENSHELL_DOCKER_SUPERVISOR_IMAGE: ghcr.io/nvidia/openshell/supervisor@sha256:b58be5e40c788977ffa0e8305a8cad9c656efdf1a3fe182582a00ca870bb0edb
steps:
- id: trusted_hermes_swap
name: Provision trusted Hermes E2E swap
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && matrix.agent == 'hermes' }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
DISPATCH_SHA: ${{ github.sha }}
ENV: /dev/null
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }}
LC_ALL: C
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
RUNNER_ARCH_KIND: ${{ runner.arch }}
RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }}
RUNNER_OS_KIND: ${{ runner.os }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
run: &trusted-hermes-e2e-swap |
set -euo pipefail
readonly swap_dir="/mnt/nemoclaw-hermes-e2e-swap"
readonly required_swap_bytes=34359738368
readonly swap_file_bytes=34359742464
readonly reserve_bytes=17179869184
readonly activation_observation_attempts=5
readonly activation_observation_delay_seconds=1
swap_file=""
swap_activation_succeeded=0
fail() {
printf 'Trusted Hermes E2E swap setup failed: %s\n' "$1" >&2
exit 1
}
if [[ "${REPOSITORY}" != "NVIDIA/NemoClaw" || "${REF}" != "refs/heads/main" ]]; then
fail "workflow must run from NVIDIA/NemoClaw main"
fi
if [[ "${EVENT_NAME}" != "push" && "${EVENT_NAME}" != "workflow_dispatch" ]]; then
fail "workflow event must be push or workflow_dispatch"
fi
# PR E2E mode: maintainer-dispatched PR commit.
if [[ "${EVENT_NAME}" == "workflow_dispatch" && -n "${CHECKOUT_SHA}" ]]; then
if [[ ! "${CHECKOUT_SHA}" =~ ^[0-9a-f]{40}$ ]]; then
fail "checkout SHA must be lowercase 40-hex"
fi
if [[ ! "${EXPECTED_WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${EXPECTED_WORKFLOW_SHA}" || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then
fail "workflow source must match the trusted dispatch revision"
fi
else
# Direct-main mode: push or manual trigger on main.
if [[ -n "${CHECKOUT_SHA}" || -n "${EXPECTED_WORKFLOW_SHA}" ]]; then
fail "direct main runs must not request an alternate checkout or workflow revision"
fi
if [[ ! "${WORKFLOW_SHA}" =~ ^[0-9a-f]{40}$ || "${WORKFLOW_SHA}" != "${DISPATCH_SHA}" ]]; then
fail "direct main workflow source must match the run revision"
fi
fi
if [[ "${RUNNER_ENVIRONMENT_KIND}" != "github-hosted" || "${RUNNER_OS_KIND}" != "Linux" || "${RUNNER_ARCH_KIND}" != "X64" ]]; then
fail "swap fallback requires an ephemeral GitHub-hosted Linux x64 runner"
fi
mnt_metadata="$(/usr/bin/stat -c "%F:%u:%g" -- /mnt)"
if [[ "${mnt_metadata}" != "directory:0:0" ]]; then
fail "/mnt must be a root-owned directory"
fi
read_active_swap_bytes() {
/usr/bin/sudo -n /usr/sbin/swapon --show=SIZE --bytes --noheadings |
/usr/bin/awk '{ total += $1 } END { printf "%.0f", total }'
}
active_swap_bytes="$(read_active_swap_bytes)"
active_swap_bytes="${active_swap_bytes:-0}"
if [[ ! "${active_swap_bytes}" =~ ^[0-9]+$ ]]; then
fail "unable to determine active swap capacity"
fi
if (( active_swap_bytes >= required_swap_bytes )); then
printf 'Hermes E2E swap is already sufficient: %s bytes active\n' "${active_swap_bytes}"
exit 0
fi
available_bytes="$(/usr/bin/df --block-size=1 --output=avail /mnt | /usr/bin/tail -n 1 | /usr/bin/tr -d "[:space:]")"
if [[ ! "${available_bytes}" =~ ^[0-9]+$ ]]; then
fail "unable to determine available disk capacity under /mnt"
fi
required_disk_bytes=$((swap_file_bytes + reserve_bytes))
if (( available_bytes < required_disk_bytes )); then
fail "insufficient disk capacity: ${available_bytes} bytes available, ${required_disk_bytes} required"
fi
if /usr/bin/sudo -n /usr/bin/test -e "${swap_dir}" || /usr/bin/sudo -n /usr/bin/test -L "${swap_dir}"; then
fail "refusing unexpected pre-existing swap path"
fi
directory_created=0
cleanup_partial_swap() {
status="$?"
if (( status != 0 && directory_created == 1 )); then
if active_swap_names="$(/usr/bin/sudo -n /usr/sbin/swapon --show=NAME --noheadings --raw 2>/dev/null)"; then
fixed_swap_active=0
while IFS= read -r active_swap_name; do
if [[ -n "${swap_file}" && "${active_swap_name}" == "${swap_file}" ]]; then
fixed_swap_active=1
break
fi
done <<< "${active_swap_names}"
if (( fixed_swap_active == 1 || swap_activation_succeeded == 1 )); then
if /usr/bin/sudo -n /usr/sbin/swapoff "${swap_file}" 2>/dev/null; then
/usr/bin/sudo -n /usr/bin/rm -f -- "${swap_file}" || true
/usr/bin/sudo -n /usr/bin/rmdir -- "${swap_dir}" || true
else
printf 'Preserving active Hermes E2E swap after setup failure: %s\n' "${swap_file}" >&2
fi
else
if [[ -n "${swap_file}" ]]; then
/usr/bin/sudo -n /usr/bin/rm -f -- "${swap_file}" || true
fi
/usr/bin/sudo -n /usr/bin/rmdir -- "${swap_dir}" || true
fi
else
printf 'Preserving Hermes E2E swap because active swap could not be queried: %s\n' "${swap_file}" >&2
fi
fi
trap - EXIT
exit "${status}"
}
trap cleanup_partial_swap EXIT
/usr/bin/sudo -n /usr/bin/mkdir -m 0700 -- "${swap_dir}"
directory_created=1
directory_metadata="$(/usr/bin/sudo -n /usr/bin/stat -c "%F:%u:%g:%a" -- "${swap_dir}")"
if [[ "${directory_metadata}" != "directory:0:0:700" ]]; then
fail "swap directory must be a root-owned mode-0700 directory"
fi
swap_file="$(/usr/bin/sudo -n /usr/bin/mktemp --tmpdir="${swap_dir}" nemoclaw-hermes.XXXXXXXX.swap)"
if ! /usr/bin/sudo -n /usr/bin/test -f "${swap_file}" || /usr/bin/sudo -n /usr/bin/test -L "${swap_file}"; then
fail "swap file must be a regular non-symlink"
fi
file_metadata="$(/usr/bin/sudo -n /usr/bin/stat -c "%u:%g:%a" -- "${swap_file}")"
if [[ "${file_metadata}" != "0:0:600" ]]; then
fail "swap file must be root-owned mode 0600"
fi
/usr/bin/sudo -n /usr/bin/fallocate -l "${swap_file_bytes}" "${swap_file}"
file_size_bytes="$(/usr/bin/sudo -n /usr/bin/stat -c "%s" -- "${swap_file}")"
if [[ ! "${file_size_bytes}" =~ ^[0-9]+$ || "${file_size_bytes}" -ne "${swap_file_bytes}" ]]; then
fail "swap file size does not match the fixed backing allocation"
fi
remaining_bytes="$(/usr/bin/df --block-size=1 --output=avail /mnt | /usr/bin/tail -n 1 | /usr/bin/tr -d "[:space:]")"
if [[ ! "${remaining_bytes}" =~ ^[0-9]+$ || "${remaining_bytes}" -lt "${reserve_bytes}" ]]; then
fail "swap allocation did not preserve the required disk reserve"
fi
/usr/bin/sudo -n /usr/sbin/mkswap --quiet "${swap_file}"
/usr/bin/sudo -n /usr/sbin/swapon "${swap_file}"
swap_activation_succeeded=1
observe_provisioned_swap() {
activation_observation_attempt=1
while (( activation_observation_attempt <= activation_observation_attempts )); do
provisioned_swap_active=0
if active_swap_names="$(/usr/bin/sudo -n /usr/sbin/swapon --show=NAME --noheadings --raw 2>/dev/null)"; then
while IFS= read -r active_swap_name; do
if [[ "${active_swap_name}" == "${swap_file}" ]]; then
provisioned_swap_active=1
break
fi
done <<< "${active_swap_names}"
fi
if observed_swap_bytes="$(read_active_swap_bytes 2>/dev/null)"; then
observed_swap_bytes="${observed_swap_bytes:-0}"
if [[ "${observed_swap_bytes}" =~ ^[0-9]+$ ]] &&
(( provisioned_swap_active == 1 && observed_swap_bytes >= required_swap_bytes )); then
active_swap_bytes="${observed_swap_bytes}"
return 0
fi
fi
if (( activation_observation_attempt < activation_observation_attempts )); then
/usr/bin/sleep "${activation_observation_delay_seconds}"
fi
activation_observation_attempt=$((activation_observation_attempt + 1))
done
return 1
}
if ! observe_provisioned_swap; then
fail "unable to verify the required active swap capacity after bounded observation"
fi
trap - EXIT
printf 'Hermes E2E swap ready: %s bytes active\n' "${active_swap_bytes}"
/usr/bin/sudo -n /usr/sbin/swapon --show
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false
- *dockerhub-auth
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@c246409193a31133cab10c8a3589001cc0d59eb3
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Initialize runner comparison telemetry
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && (matrix.agent == 'hermes' || matrix.agent == 'deepagents') }}
continue-on-error: true
shell: bash
run: npx tsx tools/e2e/runner-comparison.mts initialize
- name: Install and verify cloudflared prerequisite
# Update posture: maintainers review upstream cloudflared releases and
# update the version and reviewed SHA256 together in both explicit MCP
# lanes; mutable package repositories and unreviewed latest releases
# are intentionally rejected by the workflow-contract tests.
env:
CLOUDFLARED_VERSION: "2026.6.1"
CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526"
run: |
set -euo pipefail
cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb"
curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}"
printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c -
package="$(dpkg-deb -f "${cloudflared_deb}" Package)"
version="$(dpkg-deb -f "${cloudflared_deb}" Version)"
architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)"
if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then
printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2
exit 1
fi
sudo dpkg -i "${cloudflared_deb}"
cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}"
- name: Generate MCP test TLS
run: bash test/e2e/setup-mcp-test-tls.sh
- name: Install OpenShell CLI
env:
NEMOCLAW_OPENSHELL_FORCE_INSTALL: "1"
run: |
set -euo pipefail
bash scripts/install-openshell.sh
if [[ "$NEMOCLAW_MCP_BRIDGE_AGENT" == "deepagents" ]]; then
mkdir -p "$E2E_ARTIFACT_DIR/mcp-bridge-deepagents"
jq -n '{
schemaVersion: 1,
sourceRepository: "NVIDIA/OpenShell",
releaseTag: "v0.0.101",
sourceSha: "8ddd98c3dff62619a3963f99ba1e055b67650e72",
artifacts: {
cli: {binarySha256: "1ad48efd5e1de8f3f017a81b3a7177872f350343a1a8d8074c7e844bca4801e9"},
gateway: {binarySha256: "a6a5d754605a2144b148637b85a09291d2eeb77e08a4ee34b83685c6920448f5"},
standaloneSandbox: {binarySha256: "a2704babbb468fd0a359bfdd9844de71095b730758541b4ca8cbab77d4018920"}
}
}' > "$E2E_ARTIFACT_DIR/mcp-bridge-deepagents/openshell-exact-main-provenance.json"
fi
- name: Run MCP OpenShell provider live test
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
if command -v openshell >/dev/null 2>&1; then
OPENSHELL_BIN="$(command -v openshell)"
elif [ -x "$HOME/.local/bin/openshell" ]; then
OPENSHELL_BIN="$HOME/.local/bin/openshell"
else
echo "::error::OpenShell CLI not found after install"
exit 1
fi
export OPENSHELL_BIN
"$OPENSHELL_BIN" --version
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/mcp-bridge.test.ts
- name: Finalize runner comparison telemetry
if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' && (matrix.agent == 'hermes' || matrix.agent == 'deepagents') }}
continue-on-error: true
shell: bash
run: npx tsx tools/e2e/runner-comparison.mts finalize
- id: mcp_artifact_secret_scan
name: Scan MCP artifacts for fixture credentials
if: always()
run: >-
npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts
e2e-artifacts/live/mcp-bridge/${{ matrix.agent }}
- name: Upload MCP server artifacts
if: ${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }}
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-mcp-bridge-${{ matrix.agent }}
path: e2e-artifacts/live/mcp-bridge/${{ matrix.agent }}/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
openshell-credential-generation-window:
needs: generate-matrix
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'openshell-credential-generation-window') }}
runs-on: ubuntu-latest
permissions:
contents: read
# Keep the credential-generation lifecycle on a fresh runner so it can
# execute in parallel with, and fail independently from, the Deep Agents
# MCP lifecycle without sharing destructive sandbox state.
timeout-minutes: 90
env:
E2E_JOB: "1"
E2E_TARGET_ID: "openshell-credential-generation-window"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openshell-credential-generation-window
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_OPENSHELL_CHANNEL: stable
NEMOCLAW_OPENSHELL_EXACT_MAIN_PROOF: "1"
NEMOCLAW_RUN_LIVE_E2E: "1"
OPENSHELL_DOCKER_SUPERVISOR_IMAGE: ghcr.io/nvidia/openshell/supervisor@sha256:b58be5e40c788977ffa0e8305a8cad9c656efdf1a3fe182582a00ca870bb0edb
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false
- *dockerhub-auth
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@c246409193a31133cab10c8a3589001cc0d59eb3
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Install and verify cloudflared prerequisite
env:
CLOUDFLARED_VERSION: "2026.6.1"
CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526"
run: |
set -euo pipefail
cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb"
curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}"
printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c -
package="$(dpkg-deb -f "${cloudflared_deb}" Package)"
version="$(dpkg-deb -f "${cloudflared_deb}" Version)"
architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)"
if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then
printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2
exit 1
fi
sudo dpkg -i "${cloudflared_deb}"
cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}"
- name: Generate MCP test TLS
run: bash test/e2e/setup-mcp-test-tls.sh
- name: Install OpenShell CLI
env:
NEMOCLAW_OPENSHELL_FORCE_INSTALL: "1"
run: |
set -euo pipefail
bash scripts/install-openshell.sh
mkdir -p "$E2E_ARTIFACT_DIR/openshell-credential-generation-window"
jq -n '{
schemaVersion: 1,
sourceRepository: "NVIDIA/OpenShell",
releaseTag: "v0.0.101",
sourceSha: "8ddd98c3dff62619a3963f99ba1e055b67650e72",
artifacts: {
cli: {binarySha256: "1ad48efd5e1de8f3f017a81b3a7177872f350343a1a8d8074c7e844bca4801e9"},
gateway: {binarySha256: "a6a5d754605a2144b148637b85a09291d2eeb77e08a4ee34b83685c6920448f5"},
standaloneSandbox: {binarySha256: "a2704babbb468fd0a359bfdd9844de71095b730758541b4ca8cbab77d4018920"}
}
}' > "$E2E_ARTIFACT_DIR/openshell-credential-generation-window/openshell-exact-main-provenance.json"
- name: Run OpenShell credential generation-window live test
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
if command -v openshell >/dev/null 2>&1; then
OPENSHELL_BIN="$(command -v openshell)"
elif [ -x "$HOME/.local/bin/openshell" ]; then
OPENSHELL_BIN="$HOME/.local/bin/openshell"
else
echo "::error::OpenShell CLI not found after install"
exit 1
fi
export OPENSHELL_BIN
"$OPENSHELL_BIN" --version
npx vitest run --project e2e-live \
test/e2e/live/openshell-credential-generation-window.test.ts \
-t '^openshell-credential-generation-window$' \
--no-file-parallelism \
--silent=false --reporter=default --reporter=test/e2e/risk-signal-reporter.ts
- id: credential_window_artifact_secret_scan
name: Scan credential-window artifacts for fixture credentials
if: always()
run: >-
npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts
e2e-artifacts/live/openshell-credential-generation-window
- name: Upload credential-window artifacts
if: ${{ always() && steps.credential_window_artifact_secret_scan.outcome == 'success' }}
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-openshell-credential-generation-window
path: e2e-artifacts/live/openshell-credential-generation-window/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
openshell-dev-artifact:
needs: generate-matrix
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'mcp-bridge-dev') }}
runs-on: ubuntu-latest
permissions:
contents: read
timeout-minutes: 15
outputs:
artifact_name: ${{ steps.resolve_openshell_dev_artifact.outputs.artifact_name }}
source_commit: ${{ steps.resolve_openshell_dev_artifact.outputs.source_commit }}
manifest_sha256: ${{ steps.resolve_openshell_dev_artifact.outputs.manifest_sha256 }}
steps:
- name: Checkout trusted OpenShell dev tooling
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ github.repository }}
ref: ${{ inputs.workflow_sha || github.workflow_sha }}
path: .trusted-openshell-dev-artifact
persist-credentials: false
sparse-checkout: |
scripts/install-openshell.sh
tools/e2e/openshell-dev-artifact.mts
- name: Set up Node for OpenShell dev artifact resolution
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
- id: resolve_openshell_dev_artifact
name: Resolve immutable OpenShell dev artifact
run: >-
node --experimental-strip-types --no-warnings
"${{ github.workspace }}/.trusted-openshell-dev-artifact/tools/e2e/openshell-dev-artifact.mts" resolve
"${{ runner.temp }}/openshell-dev-artifact"
- name: Upload OpenShell dev artifact resolution
if: ${{ always() }}
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: ${{ steps.resolve_openshell_dev_artifact.outputs.artifact_name || format('openshell-dev-infrastructure-failure-{0}-{1}', github.run_id, github.run_attempt) }}
path: ${{ runner.temp }}/openshell-dev-artifact/
mcp-bridge-dev:
needs: [generate-matrix, openshell-dev-artifact]
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'mcp-bridge-dev') }}
runs-on: ubuntu-latest
permissions:
contents: read
timeout-minutes: 90
strategy:
fail-fast: false
matrix:
agent: [openclaw, hermes, deepagents]
env:
E2E_JOB: "1"
E2E_TARGET_ID: "mcp-bridge-dev"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_E2E_SHARD: ${{ matrix.agent }}
NEMOCLAW_MCP_BRIDGE_AGENT: ${{ matrix.agent }}
NEMOCLAW_OPENSHELL_CHANNEL: dev
NEMOCLAW_RUN_LIVE_E2E: "1"
steps:
# setup-node probes package managers in the workspace.
# Run it before candidate checkout with automatic caching disabled.
- name: Set up Node.js for trusted OpenShell verification
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
package-manager-cache: false
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false
- *dockerhub-auth
- name: Checkout trusted OpenShell dev tooling
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ github.repository }}
ref: ${{ inputs.workflow_sha || github.workflow_sha }}
path: .trusted-openshell-dev-artifact
persist-credentials: false
sparse-checkout: |
.github/scripts/docker-auth-cleanup.sh
scripts/install-openshell.sh
tools/e2e/openshell-dev-artifact.mts
- name: Restore immutable OpenShell dev artifact
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: ${{ needs.openshell-dev-artifact.outputs.artifact_name }}
path: ${{ runner.temp }}/openshell-dev-artifact
digest-mismatch: error
- name: Verify immutable OpenShell dev artifact
env:
OPENSHELL_DEV_ARTIFACT_DIR: ${{ runner.temp }}/openshell-dev-artifact
OPENSHELL_DEV_EXPECTED_MANIFEST_SHA256: ${{ needs.openshell-dev-artifact.outputs.manifest_sha256 }}
OPENSHELL_DEV_EXPECTED_SOURCE_COMMIT: ${{ needs.openshell-dev-artifact.outputs.source_commit }}
run: >-
node --experimental-strip-types --no-warnings
"${{ github.workspace }}/.trusted-openshell-dev-artifact/tools/e2e/openshell-dev-artifact.mts" verify
"$OPENSHELL_DEV_ARTIFACT_DIR"
"$OPENSHELL_DEV_EXPECTED_SOURCE_COMMIT"
"$OPENSHELL_DEV_EXPECTED_MANIFEST_SHA256"
- name: Revoke Docker auth before OpenShell development tooling
shell: bash
run: >-
bash
"${{ github.workspace }}/.trusted-openshell-dev-artifact/.github/scripts/docker-auth-cleanup.sh"
- name: Install immutable OpenShell dev artifact
env:
NEMOCLAW_ACCEPT_DEV_UNVERIFIED_INSTALL: "1"
NEMOCLAW_OPENSHELL_FORCE_INSTALL: "1"
OPENSHELL_DEV_ASSET_DIR: ${{ runner.temp }}/openshell-dev-artifact/assets
run: |
set -euo pipefail
shim_dir="$(mktemp -d)"
trap 'rm -rf "$shim_dir"' EXIT
cat >"$shim_dir/gh" <<'EOF'
#!/usr/bin/env bash
set -euo pipefail
if [[ "$#" -ne 10 || "$1" != "release" || "$2" != "download" || "$3" != "dev" || "$4" != "--repo" || "$5" != "NVIDIA/OpenShell" || "$6" != "--pattern" || "$8" != "--dir" || "${10}" != "--clobber" ]]; then
printf 'Unsupported gh invocation for retained OpenShell assets.\n' >&2
exit 64
fi
asset="$7"
destination="$9"
case "$asset" in
openshell-x86_64-unknown-linux-musl.tar.gz | openshell-checksums-sha256.txt | openshell-gateway-x86_64-unknown-linux-gnu.tar.gz | openshell-gateway-checksums-sha256.txt | openshell-sandbox-x86_64-unknown-linux-gnu.tar.gz | openshell-sandbox-checksums-sha256.txt) ;;
*)
printf 'Unsupported retained OpenShell asset: %s\n' "$asset" >&2
exit 64
;;
esac
source_asset="${OPENSHELL_DEV_ASSET_DIR}/${asset}"
[[ -f "$source_asset" && ! -L "$source_asset" && "$destination" = /* && -d "$destination" && ! -L "$destination" ]]
cp -- "$source_asset" "$destination/$asset"
EOF
cat >"$shim_dir/curl" <<'EOF'
#!/usr/bin/env bash
printf 'Network fallback is disabled for retained OpenShell assets.\n' >&2
exit 1
EOF
chmod 700 "$shim_dir/gh" "$shim_dir/curl"
PATH="$shim_dir:$PATH" \
bash "${{ github.workspace }}/.trusted-openshell-dev-artifact/scripts/install-openshell.sh"
# Dependency preparation can read candidate project configuration. Keep
# it after the trusted installer has consumed the verified OpenShell
# assets and Docker credentials have been revoked.
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
# The restore action executes the candidate CLI for its final identity
# check. Candidate-controlled state starts with dependency preparation.
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@c246409193a31133cab10c8a3589001cc0d59eb3
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Install and verify cloudflared prerequisite
# Update posture: keep this dev compatibility lane on the same reviewed
# version/SHA256 pair as the stable lane; workflow-contract tests fail
# if the pins diverge or installation becomes mutable.
env:
CLOUDFLARED_VERSION: "2026.6.1"
CLOUDFLARED_DEB_SHA256: "ccd02ec216c62bfa573395d8f72cb2e91e95cbdf8726a8acc06b3e2d9aa31526"
run: |
set -euo pipefail
cloudflared_deb="${RUNNER_TEMP}/cloudflared-${CLOUDFLARED_VERSION}-linux-amd64.deb"
curl -fL "https://github.com/cloudflare/cloudflared/releases/download/${CLOUDFLARED_VERSION}/cloudflared-linux-amd64.deb" -o "${cloudflared_deb}"
printf '%s %s\n' "${CLOUDFLARED_DEB_SHA256}" "${cloudflared_deb}" | sha256sum -c -
package="$(dpkg-deb -f "${cloudflared_deb}" Package)"
version="$(dpkg-deb -f "${cloudflared_deb}" Version)"
architecture="$(dpkg-deb -f "${cloudflared_deb}" Architecture)"
if [[ "${package}" != "cloudflared" || "${version}" != "${CLOUDFLARED_VERSION}" || "${architecture}" != "amd64" ]]; then
printf 'Unexpected cloudflared package metadata: package=%s version=%s architecture=%s\n' "${package}" "${version}" "${architecture}" >&2
exit 1
fi
sudo dpkg -i "${cloudflared_deb}"
cloudflared --version | grep -F "cloudflared version ${CLOUDFLARED_VERSION}"
- name: Generate MCP test TLS
run: bash test/e2e/setup-mcp-test-tls.sh
- id: mcp_runtime_compatibility
name: Classify OpenShell credential-boundary compatibility
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
npx tsx tools/e2e/mcp-bridge-runtime-compatibility.mts
- name: Run MCP OpenShell provider live test
if: ${{ steps.mcp_runtime_compatibility.outputs.mode == 'full-lifecycle' }}
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
if command -v openshell >/dev/null 2>&1; then
OPENSHELL_BIN="$(command -v openshell)"
elif [ -x "$HOME/.local/bin/openshell" ]; then
OPENSHELL_BIN="$HOME/.local/bin/openshell"
else
echo "::error::OpenShell CLI not found after install"
exit 1
fi
export OPENSHELL_BIN
"$OPENSHELL_BIN" --version
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/mcp-bridge.test.ts
- id: mcp_artifact_secret_scan
name: Scan MCP artifacts for fixture credentials
if: always()
run: >-
npx tsx tools/e2e/assert-mcp-artifact-secrets-absent.mts
e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}
- name: Upload MCP server artifacts
if: ${{ always() && steps.mcp_artifact_secret_scan.outcome == 'success' }}
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-mcp-bridge-dev-${{ matrix.agent }}
path: e2e-artifacts/live/mcp-bridge-dev/${{ matrix.agent }}/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
# Manual PR qualification also requires the exact candidate activation contract.
managed-image-multiarch-startup:
name: Protected managed-image startup (${{ matrix.platform }})
needs: generate-matrix
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'managed-image-multiarch-startup') || contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'managed-image-protected-runtime')) }}
runs-on: ${{ matrix.runner }}
timeout-minutes: 210
permissions:
contents: read
strategy:
fail-fast: false
matrix:
include:
- platform: linux/amd64
runner: ubuntu-24.04
shard: linux-amd64
- platform: linux/arm64
runner: ubuntu-24.04-arm
shard: linux-arm64
env:
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}
E2E_JOB: "1"
E2E_TARGET_ID: "managed-image-multiarch-startup"
RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-multiarch-activation-v1.json
NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_HEAD_SHA: ${{ inputs.checkout_sha || github.sha }}
NEMOCLAW_E2E_SHARD: ${{ matrix.shard }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE: ${{ github.workspace }}/.protected-managed-image-build-cache/${{ matrix.shard }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE_ARTIFACT: protected-managed-image-build-cache-${{ github.run_id }}-${{ inputs.checkout_sha || github.sha }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT: protected-${{ github.run_id }}-${{ github.run_attempt }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT: ${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}/contracts.json
NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE: ${{ github.workspace }}/e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}/evidence.json
NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM: ${{ matrix.platform }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
NEMOCLAW_PROTECTED_REGISTRY_NAME: nemoclaw-managed-${{ matrix.shard }}-${{ github.run_id }}-${{ github.run_attempt }}
NEMOCLAW_RUN_LIVE_E2E: "1"
steps:
- name: Validate protected exact-head dispatch
env:
BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
PLATFORM: ${{ matrix.platform }}
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
RUNNER_ARCH_KIND: ${{ runner.arch }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
[[ "$REPOSITORY" == "NVIDIA/NemoClaw" && "$REF" == "refs/heads/main" && ( "$EVENT_NAME" == "push" || "$EVENT_NAME" == "workflow_dispatch" ) ]] || {
echo "::error::Protected managed-image startup must run from trusted NVIDIA/NemoClaw main" >&2
exit 1
}
[[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || {
echo "::error::Protected managed-image startup requires candidate and base SHAs" >&2
exit 1
}
[[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$WORKFLOW_SHA" == "$EXPECTED_WORKFLOW_SHA" ]] || {
echo "::error::Protected managed-image startup requires the exact trusted workflow SHA" >&2
exit 1
}
case "${PLATFORM}:${RUNNER_ARCH_KIND}" in
linux/amd64:X64 | linux/arm64:ARM64) ;;
*)
echo "::error::Protected managed-image startup requires a native ${PLATFORM} runner" >&2
exit 1
;;
esac
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
fetch-depth: 0
persist-credentials: false
- *dockerhub-auth
- name: Set up protected managed-image Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
with:
driver-opts: network=host
buildkitd-config-inline: |
[registry."localhost:5000"]
http = true
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Validate candidate activation contract
env:
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
shell: bash
run: |
set -euo pipefail
activation="ci/protected-managed-image-multiarch-activation-v1.json"
[[ "$(git rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || {
echo "::error::Protected managed-image checkout does not match the candidate SHA" >&2
exit 1
}
[[ -f "$activation" && ! -L "$activation" ]] || {
echo "::error::Protected managed-image activation contract is absent" >&2
exit 1
}
jq -e '
(keys | sort) == ["agents", "contractVersion", "jobId", "platforms"] and
.contractVersion == 1 and
.jobId == "managed-image-multiarch-startup" and
.agents == ["openclaw", "hermes", "langchain-deepagents-code"] and
.platforms == ["linux/amd64", "linux/arm64"]
' "$activation" >/dev/null || {
echo "::error::Protected managed-image activation contract is invalid" >&2
exit 1
}
install -d -m 0700 "$E2E_ARTIFACT_DIR"
- id: bases
name: Resolve exact platform base images
env:
PLATFORM: ${{ matrix.platform }}
shell: bash
run: |
set -euo pipefail
arch="${PLATFORM#linux/}"
work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-protected-bases.XXXXXX")"
trap 'rm -rf -- "$work_dir"' EXIT
resolve_base() {
local output_name="$1"
local alias="$2"
local repository="$3"
local alias_raw="$work_dir/${output_name}-alias.raw"
local exact_raw="$work_dir/${output_name}-exact.raw"
docker buildx imagetools inspect "$alias" --raw > "$alias_raw"
local digest
digest="$(
jq -er --arg arch "$arch" '
if (
.mediaType == "application/vnd.oci.image.index.v1+json" or
.mediaType == "application/vnd.docker.distribution.manifest.list.v2+json"
) then
[.manifests[] | select(.platform.os == "linux" and .platform.architecture == $arch)]
| if length == 1 then .[0].digest else error("not one exact platform descriptor") end
else
error("base alias is not a platform index")
end
' "$alias_raw"
)"
[[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]] || {
echo "::error::${output_name} base alias returned an invalid digest" >&2
exit 1
}
local reference="${repository}@${digest}"
docker buildx imagetools inspect "$reference" --raw > "$exact_raw"
[[ "sha256:$(sha256sum "$exact_raw" | awk '{print $1}')" == "$digest" ]] || {
echo "::error::${output_name} exact base bytes do not match the selected digest" >&2
exit 1
}
printf '%s=%s\n' "$output_name" "$reference" >> "$GITHUB_OUTPUT"
}
resolve_base openclaw \
ghcr.io/nvidia/nemoclaw/sandbox-base:latest \
ghcr.io/nvidia/nemoclaw/sandbox-base
resolve_base hermes \
ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest \
ghcr.io/nvidia/nemoclaw/hermes-sandbox-base
resolve_base dcode \
ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base:latest \
ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base
- name: Start isolated protected managed-image registry
shell: bash
run: |
set -euo pipefail
if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then
echo "::error::Protected registry name already exists" >&2
exit 1
fi
if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then
echo "::error::Refusing to reuse an existing localhost:5000 registry" >&2
exit 1
fi
docker run --detach \
--name "$NEMOCLAW_PROTECTED_REGISTRY_NAME" \
--label "io.nvidia.nemoclaw.e2e-owner=${NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT}" \
--label "io.nvidia.nemoclaw.e2e-platform=${NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM}" \
--publish 127.0.0.1:5000:5000 \
docker.io/library/registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373
for _ in $(seq 1 30); do
if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null; then
exit 0
fi
sleep 1
done
docker logs "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >&2
exit 1
- name: Build exact all-agent protected managed images
env:
BASE_DCODE: ${{ steps.bases.outputs.dcode }}
BASE_HERMES: ${{ steps.bases.outputs.hermes }}
BASE_OPENCLAW: ${{ steps.bases.outputs.openclaw }}
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
shell: bash
run: |
set -euo pipefail
cache_args=()
if [[ "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM" == "linux/amd64" ]]; then
cache_args=(--cache-to "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE")
fi
scripts/checks/build-protected-managed-images.sh \
--output "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT" \
--revision "$CHECKOUT_SHA" \
--cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" \
--platform "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM" \
--openclaw-base "$BASE_OPENCLAW" \
--hermes-base "$BASE_HERMES" \
--dcode-base "$BASE_DCODE" \
"${cache_args[@]}"
- name: Run every exact managed-image contract directly
env:
BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
direct_runs="${RUNNER_TEMP}/protected-managed-image-direct-runs.jsonl"
: > "$direct_runs"
while IFS= read -r entry; do
agent="$(jq -er '.agent' <<< "$entry")"
digest="$(jq -er '.digest' <<< "$entry")"
platform="$(jq -er '.platform' <<< "$entry")"
reference="$(jq -er '.reference' <<< "$entry")"
[[ "$platform" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM" ]] || {
echo "::error::Protected managed-image contract changed platform" >&2
exit 1
}
npx --no-install tsx scripts/checks/run-managed-image-direct-e2e.ts \
--agent "$agent" \
--image "$reference" \
--platform "$platform"
jq -nc \
--arg agent "$agent" \
--arg digest "$digest" \
--arg platform "$platform" \
--arg reference "$reference" \
'{agent: $agent, digest: $digest, platform: $platform, reference: $reference}' \
>> "$direct_runs"
done < <(jq -c '.[]' "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT")
contract_sha="sha256:$(sha256sum "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT" | awk '{print $1}')"
jq -n \
--arg baseSha "$BASE_SHA" \
--arg cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" \
--arg contractSha256 "$contract_sha" \
--arg headSha "$CHECKOUT_SHA" \
--arg platform "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM" \
--arg workflowSha "$WORKFLOW_SHA" \
--argjson runAttempt "$GITHUB_RUN_ATTEMPT" \
--argjson runId "$GITHUB_RUN_ID" \
--slurpfile contracts "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT" \
--slurpfile directRuns "$direct_runs" \
'{
kind: "nemoclaw-protected-managed-image-multiarch-v1",
headSha: $headSha,
baseSha: $baseSha,
workflowSha: $workflowSha,
platform: $platform,
cohort: $cohort,
contractSha256: $contractSha256,
contracts: $contracts[0],
directRuns: $directRuns,
run: {id: $runId, attempt: $runAttempt}
}' > "${NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE}.tmp"
mv \
"${NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE}.tmp" \
"$NEMOCLAW_PROTECTED_MANAGED_IMAGE_EVIDENCE"
- name: Remove isolated protected managed-image registry
if: always()
shell: bash
run: |
set -euo pipefail
if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then
owner="$(
docker container inspect \
--format '{{index .Config.Labels "io.nvidia.nemoclaw.e2e-owner"}}' \
"$NEMOCLAW_PROTECTED_REGISTRY_NAME"
)"
[[ "$owner" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" ]] || {
echo "::error::Refusing to remove a registry not owned by this protected shard" >&2
exit 1
}
docker rm -f "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null
fi
if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then
echo "::error::Protected managed-image registry container remained after cleanup" >&2
exit 1
fi
if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then
echo "::error::Protected managed-image registry listener remained after cleanup" >&2
exit 1
fi
- name: Validate protected managed-image evidence
shell: bash
run: >-
npx tsx tools/e2e/live-vitest-invocation.mts run
--test-path test/e2e/live/managed-image-multiarch-startup.test.ts
- name: Publish exact amd64 protected runtime build cache
if: ${{ matrix.platform == 'linux/amd64' }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: ${{ env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE_ARTIFACT }}
path: ${{ env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE }}/
if-no-files-found: error
retention-days: 1
compression-level: 0
overwrite: true
- name: Upload protected managed-image evidence
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-managed-image-multiarch-startup-${{ matrix.shard }}
path: e2e-artifacts/live/managed-image-multiarch-startup/${{ matrix.shard }}/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
# The trusted plan compiler runs on a standard runner before the protected
# job can be assigned. Candidate workflow code is never evaluated; trusted
# main compiles the exact candidate's declarative YAML into bounded outputs.
llama-cpp-dgx-spark-plan:
name: Compile protected llama.cpp DGX Spark plan
needs: generate-matrix
if: ${{ inputs.allow_dgx_spark_runner_queue && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'llama-cpp-dgx-spark-qualification') }}
runs-on: ubuntu-24.04
timeout-minutes: 15
permissions:
contents: read
outputs:
agent_qualification_execution: ${{ steps.plan.outputs.agent_qualification_execution }}
environment: ${{ steps.plan.outputs.environment }}
execution: ${{ steps.plan.outputs.execution }}
model_host_path: ${{ steps.plan.outputs.model_host_path }}
plan: ${{ steps.plan.outputs.plan }}
plan_sha256: ${{ steps.plan.outputs.plan_sha256 }}
qualification: ${{ steps.plan.outputs.qualification }}
runner: ${{ steps.plan.outputs.runner }}
steps:
- name: Validate trusted llama.cpp plan dispatch
env:
BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
[[ "$REPOSITORY" == "NVIDIA/NemoClaw" && "$REF" == "refs/heads/main" && ( "$EVENT_NAME" == "push" || "$EVENT_NAME" == "workflow_dispatch" ) ]] || {
echo "::error::Protected llama.cpp planning must run from trusted NVIDIA/NemoClaw main" >&2
exit 1
}
[[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || {
echo "::error::Protected llama.cpp planning requires candidate and base SHAs" >&2
exit 1
}
[[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$WORKFLOW_SHA" == "$EXPECTED_WORKFLOW_SHA" ]] || {
echo "::error::Protected llama.cpp planning requires the exact trusted workflow SHA" >&2
exit 1
}
- name: Checkout trusted llama.cpp plan compiler
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ github.repository }}
ref: ${{ inputs.workflow_sha || github.workflow_sha }}
fetch-depth: 0
persist-credentials: false
- name: Checkout exact llama.cpp candidate configuration
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
path: .candidate-llama-cpp
fetch-depth: 0
persist-credentials: false
- name: Set up Node for trusted llama.cpp plan compilation
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
cache: npm
- name: Install trusted llama.cpp plan dependencies
run: npm ci --ignore-scripts
- id: plan
name: Compile exact candidate llama.cpp qualification plan
env:
CANDIDATE_ROOT: ${{ github.workspace }}/.candidate-llama-cpp
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
shell: bash
run: |
set -euo pipefail
[[ "$(git -C "$CANDIDATE_ROOT" rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || {
echo "::error::Protected llama.cpp candidate checkout does not match the candidate SHA" >&2
exit 1
}
node --experimental-strip-types --no-warnings \
scripts/checks/export-llama-cpp-dgx-spark-qualification-plan.mts \
--source-root "$CANDIDATE_ROOT"
# The opt-in flag prevents assignment to an unavailable DGX Spark runner.
# Qualification runs only when the trusted plan reports execution=enabled.
llama-cpp-dgx-spark-qualification:
name: Protected llama.cpp on NVIDIA DGX Spark
needs: [generate-matrix, llama-cpp-dgx-spark-plan]
if: ${{ inputs.allow_dgx_spark_runner_queue && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'llama-cpp-dgx-spark-qualification') && needs.llama-cpp-dgx-spark-plan.outputs.execution == 'enabled' }}
runs-on: ${{ needs.llama-cpp-dgx-spark-plan.outputs.runner }}
environment:
name: ${{ needs.llama-cpp-dgx-spark-plan.outputs.environment }}
timeout-minutes: 300
permissions:
contents: read
env:
E2E_DEFAULT_ENABLED: "0"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/llama-cpp-dgx-spark-qualification
E2E_JOB: "1"
E2E_TARGET_ID: "llama-cpp-dgx-spark-qualification"
NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }}
NEMOCLAW_E2E_SHARD: linux-arm64-gpu-dgx-spark-gb10
NEMOCLAW_LLAMA_CPP_QUALIFICATION_BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
NEMOCLAW_LLAMA_CPP_QUALIFICATION_HEAD_SHA: ${{ inputs.checkout_sha || github.sha }}
NEMOCLAW_LLAMA_CPP_QUALIFICATION_CANDIDATE_ROOT: ${{ github.workspace }}/.candidate-llama-cpp
NEMOCLAW_LLAMA_CPP_QUALIFICATION_EVIDENCE: ${{ github.workspace }}/e2e-artifacts/live/llama-cpp-dgx-spark-qualification/evidence.json
NEMOCLAW_LLAMA_CPP_QUALIFICATION_PLAN: ${{ github.workspace }}/.llama-cpp-qualification/plan.json
NEMOCLAW_LLAMA_CPP_QUALIFICATION_PLAN_SHA256: ${{ needs.llama-cpp-dgx-spark-plan.outputs.plan_sha256 }}
NEMOCLAW_LLAMA_CPP_QUALIFICATION_REGISTRY: nemoclaw-llama-cpp-${{ github.run_id }}-${{ github.run_attempt }}
NEMOCLAW_LLAMA_CPP_QUALIFICATION_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
NEMOCLAW_RUN_LIVE_E2E: "1"
RELEASE_E2E_ACTIVATION_PATH: ci/llama-cpp-dgx-spark-qualification-v1.yaml
steps:
- name: Validate protected llama.cpp exact-head dispatch
env:
BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
RUNNER_ARCH_KIND: ${{ runner.arch }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
[[ "$REPOSITORY" == "NVIDIA/NemoClaw" && "$REF" == "refs/heads/main" && ( "$EVENT_NAME" == "push" || "$EVENT_NAME" == "workflow_dispatch" ) ]] || {
echo "::error::Protected llama.cpp qualification must run from trusted NVIDIA/NemoClaw main" >&2
exit 1
}
[[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || {
echo "::error::Protected llama.cpp qualification requires candidate and base SHAs" >&2
exit 1
}
[[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$WORKFLOW_SHA" == "$EXPECTED_WORKFLOW_SHA" ]] || {
echo "::error::Protected llama.cpp qualification requires the exact trusted workflow SHA" >&2
exit 1
}
[[ "$RUNNER_ARCH_KIND" == "ARM64" ]] || {
echo "::error::Protected llama.cpp qualification requires native Linux ARM64" >&2
exit 1
}
- name: Checkout trusted llama.cpp qualification
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ github.repository }}
ref: ${{ inputs.workflow_sha || github.workflow_sha }}
fetch-depth: 0
persist-credentials: false
- name: Checkout exact llama.cpp qualification candidate
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
path: .candidate-llama-cpp
fetch-depth: 0
persist-credentials: false
- *dockerhub-auth
- name: Set up protected llama.cpp Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
with:
driver: docker
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Install OpenShell CLI for declarative OpenClaw qualification
if: ${{ needs.llama-cpp-dgx-spark-plan.outputs.agent_qualification_execution == 'enabled' }}
run: env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh
- name: Materialize trusted llama.cpp qualification plan
env:
CANDIDATE_ROOT: ${{ github.workspace }}/.candidate-llama-cpp
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
PLAN: ${{ needs.llama-cpp-dgx-spark-plan.outputs.plan }}
PLAN_SHA256: ${{ needs.llama-cpp-dgx-spark-plan.outputs.plan_sha256 }}
shell: bash
run: |
set -euo pipefail
[[ "$(git -C "$CANDIDATE_ROOT" rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || {
echo "::error::Protected llama.cpp candidate checkout does not match the candidate SHA" >&2
exit 1
}
install -d -m 0700 "$E2E_ARTIFACT_DIR"
install -d -m 0700 "$(dirname "$NEMOCLAW_LLAMA_CPP_QUALIFICATION_PLAN")"
umask 077
printf '%s' "$PLAN" > "$NEMOCLAW_LLAMA_CPP_QUALIFICATION_PLAN"
[[ "sha256:$(sha256sum "$NEMOCLAW_LLAMA_CPP_QUALIFICATION_PLAN" | awk '{print $1}')" == "$PLAN_SHA256" ]] || {
echo "::error::Protected llama.cpp qualification plan digest changed" >&2
exit 1
}
- id: qualify
name: Build and qualify exact llama.cpp candidate
env:
BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
CANDIDATE_ROOT: ${{ github.workspace }}/.candidate-llama-cpp
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
MODEL_HOST_PATH: ${{ needs.llama-cpp-dgx-spark-plan.outputs.model_host_path }}
WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
[[ "$(git rev-parse --verify HEAD)" == "$WORKFLOW_SHA" ]] || {
echo "::error::Protected llama.cpp qualification must execute trusted workflow code" >&2
exit 1
}
npx --no-install tsx scripts/checks/run-llama-cpp-dgx-spark-qualification.mts \
--base-sha "$BASE_SHA" \
--candidate-root "$CANDIDATE_ROOT" \
--head-sha "$CHECKOUT_SHA" \
--model-host-path "$MODEL_HOST_PATH" \
--output "$NEMOCLAW_LLAMA_CPP_QUALIFICATION_EVIDENCE" \
--plan "$NEMOCLAW_LLAMA_CPP_QUALIFICATION_PLAN" \
--plan-sha256 "$NEMOCLAW_LLAMA_CPP_QUALIFICATION_PLAN_SHA256" \
--registry-name "$NEMOCLAW_LLAMA_CPP_QUALIFICATION_REGISTRY" \
--run-attempt "$GITHUB_RUN_ATTEMPT" \
--run-id "$GITHUB_RUN_ID" \
--workflow-sha "$WORKFLOW_SHA"
- name: Remove protected llama.cpp qualification resources
if: always()
shell: bash
run: >-
npx --no-install tsx scripts/checks/run-llama-cpp-dgx-spark-qualification.mts
--cleanup-only
--registry-name "$NEMOCLAW_LLAMA_CPP_QUALIFICATION_REGISTRY"
--run-attempt "$GITHUB_RUN_ATTEMPT"
--run-id "$GITHUB_RUN_ID"
- name: Validate protected llama.cpp evidence
shell: bash
run: >-
npx --no-install tsx tools/e2e/live-vitest-invocation.mts run
--test-path test/e2e/live/llama-cpp-dgx-spark-qualification.test.ts
- name: Upload protected llama.cpp evidence
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-llama-cpp-dgx-spark-qualification
path: e2e-artifacts/live/llama-cpp-dgx-spark-qualification/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
# This lane runs after managed-image startup succeeds. Manual PR qualification
# requires the exact candidate activation contract. Its
# hosted amd64 prerequisite exports the exact candidate build cache.
# This GPU lane imports the exact candidate build cache and its complete
# locked npm cache seed, then rebuilds with BuildKit networking disabled.
# invalidState: Cold managed-image builds on the protected GPU runner can
# stall in npm and terminate with "Exit handler never called!" before any
# GPU, Ollama, NIM, or vLLM assertion executes.
# sourceBoundary: The hosted amd64 lane builds the exact candidate. It adds
# every reachable Linux x64 glibc registry archive from the candidate
# lockfile to the same-run cache artifact. The GPU lane verifies the manifest,
# lock digest, archive set, and
# each SHA-512 integrity before the candidate build can consume the seed.
# BuildKit records can accelerate the rebuild, while the seed makes a missing
# npm layer reproducible without weakening the fail-closed network boundary.
# whyNotBuildCacheOnly: Protected runs 31156103003, 31158136234, and
# 31159822518 proved that network-key drift and incomplete local cache exports
# can miss the npm layer before runtime qualification.
# regressionTest: The materializer and build-script behavior tests reject an
# incomplete, substituted, unexpected, or symlinked archive handoff and pin
# exact --cache-from/--network none arguments. Workflow boundary tests pin the
# same-run cache artifact.
# lifecycleDecision: PR #8261 owns removal after three consecutive protected
# GPU cold builds of all three images reach GPU, Ollama, NIM, and vLLM
# assertions without the hosted cache.
managed-image-protected-runtime:
name: Protected managed-image GPU and local inference
needs: [generate-matrix, managed-image-multiarch-startup]
if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && needs['generate-matrix'].result == 'success' && needs['managed-image-multiarch-startup'].result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'managed-image-protected-runtime') }}
runs-on: linux-amd64-gpu-rtxpro6000-latest-1
timeout-minutes: 300
permissions:
contents: read
env:
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime
E2E_JOB: "1"
E2E_TARGET_ID: "managed-image-protected-runtime"
E2E_WORKLOAD_SOURCE: "managed-image"
RELEASE_E2E_ACTIVATION_PATH: ci/protected-managed-image-runtime-activation-v1.json
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_E2E_EXPECTED_SHA: ${{ inputs.checkout_sha }}
NEMOCLAW_E2E_SHARD: linux-amd64-gpu
NEMOCLAW_E2E_TESTED_ROOT: ${{ github.workspace }}/.candidate-runtime
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_PROTECTED_MANAGED_IMAGE_BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE: ${{ github.workspace }}/.protected-managed-image-build-cache/linux-amd64
NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE_ARTIFACT: protected-managed-image-build-cache-${{ github.run_id }}-${{ inputs.checkout_sha || github.sha }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT: protected-${{ github.run_id }}-${{ github.run_attempt }}
NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT: ${{ github.workspace }}/e2e-artifacts/live/managed-image-protected-runtime/contracts.json
NEMOCLAW_PROTECTED_MANAGED_IMAGE_PLATFORM: linux/amd64
NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
NEMOCLAW_PROTECTED_REGISTRY_NAME: nemoclaw-managed-runtime-${{ github.run_id }}-${{ github.run_attempt }}
NEMOCLAW_RUN_LIVE_E2E: "1"
OPENSHELL_GATEWAY: nemoclaw
steps:
- name: Validate protected runtime exact-head dispatch
env:
BASE_SHA: ${{ inputs.base_sha || github.event.before || github.sha }}
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha || github.workflow_sha }}
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
RUNNER_ARCH_KIND: ${{ runner.arch }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
shell: bash
run: |
set -euo pipefail
[[ "$REPOSITORY" == "NVIDIA/NemoClaw" && "$REF" == "refs/heads/main" && ( "$EVENT_NAME" == "push" || "$EVENT_NAME" == "workflow_dispatch" ) ]] || {
echo "::error::Protected managed-image runtime must run from trusted NVIDIA/NemoClaw main" >&2
exit 1
}
[[ "$CHECKOUT_SHA" =~ ^[a-f0-9]{40}$ && "$BASE_SHA" =~ ^[a-f0-9]{40}$ ]] || {
echo "::error::Protected managed-image runtime requires candidate and base SHAs" >&2
exit 1
}
[[ "$EXPECTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ && "$WORKFLOW_SHA" == "$EXPECTED_WORKFLOW_SHA" ]] || {
echo "::error::Protected managed-image runtime requires the exact trusted workflow SHA" >&2
exit 1
}
[[ "$RUNNER_ARCH_KIND" == "X64" ]] || {
echo "::error::Protected managed-image runtime requires a native linux/amd64 GPU runner" >&2
exit 1
}
- name: Checkout trusted protected runtime qualification
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ github.repository }}
ref: ${{ inputs.workflow_sha || github.workflow_sha }}
fetch-depth: 0
persist-credentials: false
- name: Checkout exact protected runtime candidate source
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
path: .candidate-runtime
fetch-depth: 0
persist-credentials: false
- name: Download exact protected runtime build cache
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
name: ${{ env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE_ARTIFACT }}
path: ${{ env.NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE }}
- *dockerhub-auth
- name: Set up protected runtime Buildx
uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
with:
driver-opts: network=host
buildkitd-config-inline: |
[registry."localhost:5000"]
http = true
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
- name: Validate protected runtime activation contract
env:
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
shell: bash
run: |
set -euo pipefail
candidate_root=".candidate-runtime"
activation="$candidate_root/ci/protected-managed-image-runtime-activation-v1.json"
[[ "$(git -C "$candidate_root" rev-parse --verify HEAD)" == "$CHECKOUT_SHA" ]] || {
echo "::error::Protected managed-image runtime checkout does not match the candidate SHA" >&2
exit 1
}
[[ -f "$activation" && ! -L "$activation" ]] || {
echo "::error::Protected managed-image runtime activation contract is absent" >&2
exit 1
}
jq -e '
(keys | sort) == ["agents", "contractVersion", "jobId", "platform", "providers"] and
.contractVersion == 1 and
.jobId == "managed-image-protected-runtime" and
.agents == ["openclaw", "hermes", "langchain-deepagents-code"] and
.platform == "linux/amd64" and
.providers == ["ollama", "nim", "vllm"]
' "$activation" >/dev/null || {
echo "::error::Protected managed-image runtime activation contract is invalid" >&2
exit 1
}
install -d -m 0700 "$E2E_ARTIFACT_DIR"
- id: runtime-bases
name: Resolve exact amd64 runtime base images
shell: bash
run: |
set -euo pipefail
work_dir="$(mktemp -d "${RUNNER_TEMP}/nemoclaw-runtime-bases.XXXXXX")"
trap 'rm -rf -- "$work_dir"' EXIT
resolve_base() {
local output_name="$1"
local alias="$2"
local repository="$3"
local alias_raw="$work_dir/${output_name}-alias.raw"
local exact_raw="$work_dir/${output_name}-exact.raw"
docker buildx imagetools inspect "$alias" --raw > "$alias_raw"
local digest
digest="$(
jq -er '
if (
.mediaType == "application/vnd.oci.image.index.v1+json" or
.mediaType == "application/vnd.docker.distribution.manifest.list.v2+json"
) then
[.manifests[] | select(.platform.os == "linux" and .platform.architecture == "amd64")]
| if length == 1 then .[0].digest else error("not one exact amd64 descriptor") end
else
error("base alias is not a platform index")
end
' "$alias_raw"
)"
[[ "$digest" =~ ^sha256:[a-f0-9]{64}$ ]] || {
echo "::error::${output_name} base alias returned an invalid digest" >&2
exit 1
}
local reference="${repository}@${digest}"
docker buildx imagetools inspect "$reference" --raw > "$exact_raw"
[[ "sha256:$(sha256sum "$exact_raw" | awk '{print $1}')" == "$digest" ]] || {
echo "::error::${output_name} exact base bytes do not match the selected digest" >&2
exit 1
}
printf '%s=%s\n' "$output_name" "$reference" >> "$GITHUB_OUTPUT"
}
resolve_base openclaw \
ghcr.io/nvidia/nemoclaw/sandbox-base:latest \
ghcr.io/nvidia/nemoclaw/sandbox-base
resolve_base hermes \
ghcr.io/nvidia/nemoclaw/hermes-sandbox-base:latest \
ghcr.io/nvidia/nemoclaw/hermes-sandbox-base
resolve_base dcode \
ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base:latest \
ghcr.io/nvidia/nemoclaw/langchain-deepagents-code-sandbox-base
- name: Start isolated protected runtime registry
shell: bash
run: |
set -euo pipefail
if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then
echo "::error::Protected runtime registry name already exists" >&2
exit 1
fi
if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then
echo "::error::Refusing to reuse an existing localhost:5000 registry" >&2
exit 1
fi
docker run --detach \
--name "$NEMOCLAW_PROTECTED_REGISTRY_NAME" \
--label "io.nvidia.nemoclaw.e2e-owner=${NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT}" \
--label "io.nvidia.nemoclaw.e2e-platform=linux/amd64" \
--publish 127.0.0.1:5000:5000 \
docker.io/library/registry@sha256:a3d8aaa63ed8681a604f1dea0aa03f100d5895b6a58ace528858a7b332415373
for _ in $(seq 1 30); do
if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null; then
exit 0
fi
sleep 1
done
docker logs "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >&2
exit 1
- name: Build exact all-agent protected runtime images
env:
BASE_DCODE: ${{ steps.runtime-bases.outputs.dcode }}
BASE_HERMES: ${{ steps.runtime-bases.outputs.hermes }}
BASE_OPENCLAW: ${{ steps.runtime-bases.outputs.openclaw }}
CHECKOUT_SHA: ${{ inputs.checkout_sha || github.sha }}
shell: bash
run: |
set -euo pipefail
scripts/checks/build-protected-managed-images.sh \
--output "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_CONTRACT" \
--revision "$CHECKOUT_SHA" \
--cohort "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" \
--platform linux/amd64 \
--source-root "$GITHUB_WORKSPACE/.candidate-runtime" \
--cache-from "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_BUILD_CACHE" \
--openclaw-base "$BASE_OPENCLAW" \
--hermes-base "$BASE_HERMES" \
--dcode-base "$BASE_DCODE"
- name: Install OpenShell CLI
shell: bash
run: env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN -u NVIDIA_API_KEY -u NVIDIA_INFERENCE_API_KEY -u GITHUB_TOKEN bash scripts/install-openshell.sh
- name: Run all-agent GPU, local inference, rollback, and cleanup qualification
env:
NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
shell: bash
run: |
set -euo pipefail
[[ "$(git rev-parse --verify HEAD)" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_WORKFLOW_SHA" ]] || {
echo "::error::Protected NIM qualification must execute trusted workflow code" >&2
exit 1
}
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
export OPENSHELL_BIN="$(command -v openshell)"
"$OPENSHELL_BIN" --version
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/managed-image-protected-runtime.test.ts
- name: Remove isolated protected runtime registry
if: always()
shell: bash
run: |
set -euo pipefail
if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then
owner="$(
docker container inspect \
--format '{{index .Config.Labels "io.nvidia.nemoclaw.e2e-owner"}}' \
"$NEMOCLAW_PROTECTED_REGISTRY_NAME"
)"
[[ "$owner" == "$NEMOCLAW_PROTECTED_MANAGED_IMAGE_COHORT" ]] || {
echo "::error::Refusing to remove a runtime registry not owned by this protected job" >&2
exit 1
}
docker rm -f "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null
fi
if docker container inspect "$NEMOCLAW_PROTECTED_REGISTRY_NAME" >/dev/null 2>&1; then
echo "::error::Protected managed-image runtime registry remained after cleanup" >&2
exit 1
fi
if curl --fail --silent --show-error http://127.0.0.1:5000/v2/ >/dev/null 2>&1; then
echo "::error::Protected managed-image runtime registry listener remained after cleanup" >&2
exit 1
fi
- name: Upload protected managed-image runtime artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-managed-image-protected-runtime
path: e2e-artifacts/live/managed-image-protected-runtime/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
hermes-e2e:
needs: generate-matrix
if: ${{ needs.generate-matrix.outputs.hermes_selected == 'true' }}
runs-on: ${{ fromJSON(needs.generate-matrix.outputs.runner_routing)['hermes-e2e'] }}
timeout-minutes: 85
env:
E2E_JOB: "1"
E2E_TARGET_ID: "hermes-e2e"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-e2e
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_INFERENCE_MODE: ${{ inputs.inference_mode || 'mock' }}
NEMOCLAW_E2E_HERMES_DASHBOARD: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_AGENT: hermes
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_RECREATE_SANDBOX: "1"
NEMOCLAW_SANDBOX_NAME: e2e-hermes
NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60"
steps:
- id: trusted_hermes_swap
name: Provision trusted Hermes E2E swap
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && (github.event_name == 'push' || inputs.checkout_sha == '' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha != '' && (contains(format(',{0},', inputs.jobs), ',hermes-e2e,') || contains(format(',{0},', inputs.targets), ',hermes-e2e,') || contains(format(',{0},', inputs.jobs), ',hermes-dashboard,') || contains(format(',{0},', inputs.targets), ',hermes-dashboard,')))) }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
CHECKOUT_SHA: ${{ inputs.checkout_sha }}
DISPATCH_SHA: ${{ github.sha }}
ENV: /dev/null
EVENT_NAME: ${{ github.event_name }}
EXPECTED_WORKFLOW_SHA: ${{ inputs.workflow_sha }}
LC_ALL: C
REF: ${{ github.ref }}
REPOSITORY: ${{ github.repository }}
RUNNER_ARCH_KIND: ${{ runner.arch }}
RUNNER_ENVIRONMENT_KIND: ${{ runner.environment }}
RUNNER_OS_KIND: ${{ runner.os }}
WORKFLOW_SHA: ${{ github.workflow_sha }}
run: *trusted-hermes-e2e-swap
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false
- *dockerhub-auth
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@c246409193a31133cab10c8a3589001cc0d59eb3
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Initialize runner comparison telemetry
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }}
continue-on-error: true
shell: bash
run: npx tsx tools/e2e/runner-comparison.mts initialize
- name: Run Hermes live Vitest test
env:
NVIDIA_INFERENCE_API_KEY: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && (inputs.inference_mode || 'mock') != 'mock' && secrets.NVIDIA_INFERENCE_API_KEY || '' }}
run: |
set -euo pipefail
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/hermes-e2e.test.ts
- name: Finalize runner comparison telemetry
if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && inputs.checkout_sha == '' }}
continue-on-error: true
shell: bash
run: npx tsx tools/e2e/runner-comparison.mts finalize
- name: Upload Hermes live Vitest artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
hermes-gpu-startup:
needs: generate-matrix
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'hermes-gpu-startup') }}
runs-on: linux-amd64-gpu-rtxpro6000-latest-1
timeout-minutes: 90
strategy:
fail-fast: false
max-parallel: 1
matrix:
include:
- scenario: native
sandbox_name: e2e-hgpu-native
- scenario: fallback
sandbox_name: e2e-hgpu-fallback
- scenario: compatibility-only
sandbox_name: e2e-hgpu-compat
env:
E2E_JOB: "1"
E2E_TARGET_ID: "hermes-gpu-startup"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}
E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }}
NEMOCLAW_E2E_SHARD: ${{ matrix.scenario }}
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_AGENT: hermes
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_RECREATE_SANDBOX: "1"
NEMOCLAW_SANDBOX_GPU: "1"
NEMOCLAW_SANDBOX_NAME: ${{ matrix.sandbox_name }}
NEMOCLAW_ONBOARD_VALIDATION_TIMEOUT_SECONDS: "60"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false
- name: Checkout trusted Hermes GPU runtime fixture
if: ${{ matrix.scenario == 'fallback' }}
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: NVIDIA/NemoClaw
ref: ${{ github.workflow_sha }}
path: .trusted-hermes-gpu-fixture-${{ github.run_id }}-${{ github.run_attempt }}
sparse-checkout: tools/e2e/hermes-gpu-docker-runtime-fixture.sh
sparse-checkout-cone-mode: false
persist-credentials: false
- name: Install trusted Hermes GPU runtime fixture
if: ${{ matrix.scenario == 'fallback' }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }}
ENV: /dev/null
TRUSTED_DISPATCH_SHA: ${{ github.sha }}
TRUSTED_FIXTURE_SHA256: e273c4baa7fe89546d64517cf56eafec30aeda7b355971263605ab1327fade02
TRUSTED_WORKFLOW_SHA: ${{ github.workflow_sha }}
run: |
set -euo pipefail
trusted_checkout="$GITHUB_WORKSPACE/.trusted-hermes-gpu-fixture-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"
trusted_source="$trusted_checkout/tools/e2e/hermes-gpu-docker-runtime-fixture.sh"
trusted_fixture="/usr/local/libexec/nemoclaw/hermes-gpu-docker-runtime-fixture.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.${E2E_HERMES_GPU_STARTUP_SCENARIO}"
trusted_state_root=/var/lib/nemoclaw-e2e
run_trusted_fixture() {
/usr/bin/sudo -n /usr/bin/env -i \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
/bin/bash "$trusted_fixture" "$@"
}
[[ "$TRUSTED_WORKFLOW_SHA" =~ ^[a-f0-9]{40}$ ]] \
|| { echo "Trusted workflow SHA must be an immutable commit" >&2; exit 1; }
[[ "$TRUSTED_DISPATCH_SHA" = "$TRUSTED_WORKFLOW_SHA" ]] \
|| { echo "Trusted fixture must match the dispatched main workflow" >&2; exit 1; }
[[ "$TRUSTED_FIXTURE_SHA256" =~ ^[a-f0-9]{64}$ ]] \
|| { echo "Trusted fixture SHA-256 must be pinned" >&2; exit 1; }
[ "$(/usr/bin/git -C "$trusted_checkout" rev-parse HEAD)" = "$TRUSTED_WORKFLOW_SHA" ] \
|| { echo "Trusted fixture checkout does not match the workflow SHA" >&2; exit 1; }
[ -f "$trusted_source" ] && [ ! -L "$trusted_source" ] \
|| { echo "Trusted Docker fixture must be a regular non-symlink file" >&2; exit 1; }
/usr/bin/sudo /usr/bin/install -d -o root -g root -m 0755 /usr/local/libexec/nemoclaw
/usr/bin/sudo /usr/bin/install -o root -g root -m 0500 \
"$trusted_source" "$trusted_fixture"
[ "$(/usr/bin/sudo /usr/bin/stat -c '%a %u %g' "$trusted_fixture")" = "500 0 0" ] \
|| { echo "Trusted Docker fixture ownership or mode is invalid" >&2; exit 1; }
printf '%s %s\n' "$TRUSTED_FIXTURE_SHA256" "$trusted_fixture" \
| /usr/bin/sudo /usr/bin/sha256sum -c -
/usr/bin/sudo /usr/bin/cmp -s "$trusted_source" "$trusted_fixture" \
|| { echo "Installed Docker fixture does not match trusted workflow code" >&2; exit 1; }
/usr/bin/sudo /usr/bin/install -d -o root -g root -m 0700 "$trusted_state_root"
# Recover any root-owned snapshot left by a hard-cancelled earlier run before
# PR-controlled build or test code executes on this persistent GPU runner.
if ! /usr/bin/sudo /usr/bin/find "$trusted_state_root" -mindepth 1 -maxdepth 1 \
-type d -name 'hermes-gpu-fallback-docker-runtime.*' -print0 \
| while IFS= read -r -d '' stale_state_dir; do
if ! run_trusted_fixture restore \
"$stale_state_dir" /etc/docker/daemon.json >/dev/null; then
exit 1
fi
done; then
echo "Could not recover stale Docker fallback state" >&2
exit 1
fi
- *dockerhub-auth
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@c246409193a31133cab10c8a3589001cc0d59eb3
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Reassert trusted Node runtime
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
env:
BASH_ENV: /dev/null
E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }}
ENV: /dev/null
NODE_OPTIONS: ""
with:
node-version: "22"
- name: Run Hermes GPU startup live Vitest test
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }}
ENV: /dev/null
run: |
set -euo pipefail
if [ "$E2E_HERMES_GPU_STARTUP_SCENARIO" = fallback ]; then
umask 077
daemon_json=/etc/docker/daemon.json
trusted_fixture="/usr/local/libexec/nemoclaw/hermes-gpu-docker-runtime-fixture.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.${E2E_HERMES_GPU_STARTUP_SCENARIO}"
trusted_state_root=/var/lib/nemoclaw-e2e
run_trusted_fixture() {
/usr/bin/sudo -n /usr/bin/env -i \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
/bin/bash "$trusted_fixture" "$@"
}
mkdir -p "$E2E_ARTIFACT_DIR"
[ "$(/usr/bin/sudo /usr/bin/stat -c '%a %u %g' "$trusted_fixture")" = "500 0 0" ] \
|| { echo "Trusted Docker fixture ownership or mode changed" >&2; exit 1; }
/usr/bin/sudo /usr/bin/install -d -o root -g root -m 0700 "$trusted_state_root"
state_dir="$(/usr/bin/sudo /usr/bin/mktemp -d \
"$trusted_state_root/hermes-gpu-fallback-docker-runtime.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.fallback.XXXXXX")"
/usr/bin/sudo /usr/bin/chown root:root "$state_dir"
/usr/bin/sudo /usr/bin/chmod 0700 "$state_dir"
# SOURCE_OF_TRUTH_REVIEW (Hermes GPU fallback fixture; #6110):
# invalidState: a cancelled fallback test leaves the runner-global Docker default
# runtime changed, contaminating later OpenShell jobs on this self-hosted runner.
# sourceBoundary: daemon mutation uses only immutable root-owned workflow_sha
# code; the PR test cannot replace the helper or its root-owned snapshot state.
# whyNotSourceFix: production must fail closed when GPU attachment is ambiguous;
# this fixture needs a real no-GPU partial container on an nvidia-default runner.
# regressionTest: hermes-gpu-startup-workflow-boundary requires this same-step trap.
# removalCondition: remove the daemon fixture when OpenShell can create a provably
# GPU-unattached partial sandbox without consulting Docker's default runtime.
restore_docker_default_runtime() {
local command_status=$?
local restore_status=0
local restored_runtime=""
trap - EXIT INT TERM
set +e
restored_runtime="$(run_trusted_fixture restore "$state_dir" "$daemon_json")"
restore_status=$?
if [ "$restore_status" -eq 0 ] && [ -n "$restored_runtime" ]; then
printf '%s\n' "$restored_runtime" \
>"$E2E_ARTIFACT_DIR/docker-default-runtime-restored.txt" \
|| restore_status=1
fi
if [ "$restore_status" -ne 0 ]; then
exit 1
fi
exit "$command_status"
}
# OpenShell leaves Runtime unset after the wrapper strips --gpu, so Docker
# otherwise substitutes its daemon default. Force runc only within this
# trap-guarded process so production can prove the partial container has no GPU.
trap restore_docker_default_runtime EXIT
trap 'exit 130' INT
trap 'exit 143' TERM
original_runtime="$(run_trusted_fixture capture "$state_dir" "$daemon_json")"
printf '%s\n' "$original_runtime" \
>"$E2E_ARTIFACT_DIR/docker-default-runtime-before.txt"
selected_runtime="$(run_trusted_fixture select-runc "$state_dir" "$daemon_json")"
printf '%s\n' "$selected_runtime" \
>"$E2E_ARTIFACT_DIR/docker-default-runtime-during.txt"
fi
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/hermes-gpu-startup.test.ts
- name: Recover Docker daemon after Hermes GPU fallback fixture
if: always()
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }}
ENV: /dev/null
run: |
set -euo pipefail
if [ "$E2E_HERMES_GPU_STARTUP_SCENARIO" != fallback ]; then
exit 0
fi
recovery_failed=0
restored_runtime=""
trusted_fixture="/usr/local/libexec/nemoclaw/hermes-gpu-docker-runtime-fixture.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.${E2E_HERMES_GPU_STARTUP_SCENARIO}"
trusted_state_root=/var/lib/nemoclaw-e2e
state_prefix="hermes-gpu-fallback-docker-runtime.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.fallback."
run_trusted_fixture() {
/usr/bin/sudo -n /usr/bin/env -i \
PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
/bin/bash "$trusted_fixture" "$@"
}
if ! /usr/bin/sudo /usr/bin/find "$trusted_state_root" -mindepth 1 -maxdepth 1 \
-type d -name "${state_prefix}*" -print0 \
| while IFS= read -r -d '' state_dir; do
if ! restored_runtime="$(run_trusted_fixture restore \
"$state_dir" /etc/docker/daemon.json)"; then
exit 1
fi
if [ -n "$restored_runtime" ]; then
printf '%s\n' "$restored_runtime" \
>"$E2E_ARTIFACT_DIR/docker-default-runtime-restored.txt"
fi
done; then
recovery_failed=1
fi
if [ "$recovery_failed" -ne 0 ]; then
echo "Independent Docker daemon recovery could not prove restoration" >&2
exit 1
fi
- name: Remove trusted Hermes GPU runtime fixture
if: ${{ always() && matrix.scenario == 'fallback' }}
shell: /bin/bash --noprofile --norc -e -o pipefail {0}
env:
BASH_ENV: /dev/null
E2E_HERMES_GPU_STARTUP_SCENARIO: ${{ matrix.scenario }}
ENV: /dev/null
run: |
set -euo pipefail
trusted_fixture="/usr/local/libexec/nemoclaw/hermes-gpu-docker-runtime-fixture.${GITHUB_RUN_ID}.${GITHUB_RUN_ATTEMPT}.${E2E_HERMES_GPU_STARTUP_SCENARIO}"
/usr/bin/sudo /usr/bin/rm -f -- "$trusted_fixture"
- name: Upload Hermes GPU startup artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-hermes-gpu-startup-${{ matrix.scenario }}
path: e2e-artifacts/live/hermes-gpu-startup/${{ matrix.scenario }}/
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
jetson-nvmap-gpu:
needs: generate-matrix
if: ${{ github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.allow_jetson_dispatch && (inputs.checkout_repository == '' || inputs.checkout_repository == github.repository) && ((inputs.jobs == '' && inputs.targets == '') || contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'jetson-nvmap-gpu')))) }}
concurrency:
group: jetson-nvmap-gpu-dispatch
queue: max
cancel-in-progress: false
runs-on: ubuntu-latest
timeout-minutes: 60
permissions:
contents: read
id-token: write
steps:
- name: Check out trusted Jetson controller
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: NVIDIA/NemoClaw
ref: ${{ github.workflow_sha }}
persist-credentials: false
- name: Set up Node for Jetson controller
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: 22
- name: Dispatch exact commit to Jetson through operator backend
env:
E2E_ARTIFACT_DIR: ${{ runner.temp }}/e2e-artifacts/live/jetson-nvmap-gpu
JETSON_DISPATCH_CANDIDATE_SHA: ${{ inputs.checkout_sha || github.sha }}
JETSON_DISPATCH_URL: ${{ vars.JETSON_DISPATCH_URL }}
run: node --experimental-strip-types --no-warnings tools/e2e/jetson-dispatch-client.mts
- name: Upload Jetson nvmap GPU artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-jetson-nvmap-gpu
path: ${{ runner.temp }}/e2e-artifacts/live/jetson-nvmap-gpu/
cloud-onboard:
needs: generate-matrix
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'cloud-onboard') }}
runs-on: ubuntu-latest
timeout-minutes: 70
env:
E2E_JOB: "1"
E2E_TARGET_ID: "cloud-onboard"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/cloud-onboard
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_SANDBOX_NAME: "e2e-cloud-onboard"
NEMOCLAW_PUBLIC_INSTALL_REF: "${{ inputs.checkout_sha || github.sha }}"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false
- *dockerhub-auth
- name: Configure cloud-onboard trace directory
shell: bash
run: |
set -euo pipefail
printf 'NEMOCLAW_TRACE_DIR=%s\n' "${RUNNER_TEMP}/nemoclaw-cloud-onboard-traces" >> "${GITHUB_ENV}"
# invalidState: the cloud-onboard DCode TUI check requires a PTY driver,
# but the fixed GitHub-hosted runner image does not provide expect.
# sourceBoundary: privileged host setup runs from the first-party
# host-dependency-setup action pinned to an immutable full SHA, never the
# repository target ref; the check only consumes expect without privilege.
# whyNotSourceFix: GitHub-hosted jobs cannot use a repository-owned host
# image, and caching privileged dpkg state between clean runners is not
# supported.
# regressionTest: workflow-boundary tests pin the ordering, action
# provenance, and package mapping.
# removalCondition: remove when the hosted runner supplies expect or the
# cloud-onboard acceptance check no longer requires a PTY.
- name: Install cloud-onboard DCode TUI host dependencies
uses: NVIDIA/NemoClaw/.github/actions/host-dependency-setup@4def1501b34ce586f83b91af50a66b5d22b31d75
with:
packages: expect
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@c246409193a31133cab10c8a3589001cc0d59eb3
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Install OpenShell CLI
run: bash scripts/install-openshell.sh
- name: Run cloud-onboard live Vitest test
env:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
run: |
set -euo pipefail
export PATH="$HOME/.local/bin:$HOME/.npm-global/bin:$PATH"
if command -v openshell >/dev/null 2>&1; then
OPENSHELL_BIN="$(command -v openshell)"
elif [ -x "$HOME/.local/bin/openshell" ]; then
OPENSHELL_BIN="$HOME/.local/bin/openshell"
else
echo "::error::OpenShell CLI not found after install"
exit 1
fi
export OPENSHELL_BIN
"$OPENSHELL_BIN" --version
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/cloud-onboard.test.ts
- name: Build trusted cloud-onboard timing summary
if: always()
shell: bash
run: |
set -euo pipefail
expected_trace_dir="${RUNNER_TEMP}/nemoclaw-cloud-onboard-traces"
if [ -z "${RUNNER_TEMP}" ] || [ "${NEMOCLAW_TRACE_DIR}" != "${expected_trace_dir}" ]; then
echo "::error::Refusing to sanitize unexpected raw trace path" >&2
exit 1
fi
python3 scripts/e2e/sanitize-trace-timing.py \
"${NEMOCLAW_TRACE_DIR}" \
"${E2E_ARTIFACT_DIR}"
# The target process must emit full local traces for diagnosis, but those
# traces may contain prompts, environment data, and credential material.
# Keep cleanup as a separate always() step so a sanitizer failure cannot
# bypass it. A runner-level termination is contained by the ephemeral
# GitHub-hosted runner.temp boundary. Remove this step only when the trace
# producer itself emits the allowlisted timing-only schema.
- name: Delete raw cloud-onboard traces
if: always()
shell: bash
run: |
set -euo pipefail
expected_trace_dir="${RUNNER_TEMP}/nemoclaw-cloud-onboard-traces"
if [ -z "${RUNNER_TEMP}" ] || [ "${NEMOCLAW_TRACE_DIR}" != "${expected_trace_dir}" ]; then
echo "::error::Refusing to delete unexpected raw trace path" >&2
exit 1
fi
rm -rf -- "${NEMOCLAW_TRACE_DIR}"
- name: Upload cloud-onboard artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
messaging-providers:
needs: generate-matrix
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'messaging-providers') }}
runs-on: ubuntu-latest
timeout-minutes: 90
env:
E2E_JOB: "1"
E2E_TARGET_ID: "messaging-providers"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/messaging-providers
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_E2E_USE_HOSTED_INFERENCE: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
OPENSHELL_GATEWAY: "nemoclaw"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false
- *dockerhub-auth
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@c246409193a31133cab10c8a3589001cc0d59eb3
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Run messaging providers live Vitest test
# The test keeps
# the legacy fake-token defaults, optional _REAL secret overrides,
# provider placeholder redaction checks, and WhatsApp QR-only parity.
env:
NVIDIA_INFERENCE_API_KEY: ${{ secrets.NVIDIA_INFERENCE_API_KEY }}
TELEGRAM_BOT_TOKEN_REAL: ${{ secrets.TELEGRAM_BOT_TOKEN_REAL }}
DISCORD_BOT_TOKEN_REAL: ${{ secrets.DISCORD_BOT_TOKEN_REAL }}
SLACK_BOT_TOKEN_REAL: ${{ secrets.SLACK_BOT_TOKEN_REAL }}
SLACK_APP_TOKEN_REAL: ${{ secrets.SLACK_APP_TOKEN_REAL }}
TELEGRAM_ALLOWED_IDS: ${{ secrets.TELEGRAM_ALLOWED_IDS }}
TELEGRAM_AUTHORIZED_CHAT_IDS: ${{ secrets.TELEGRAM_AUTHORIZED_CHAT_IDS }}
TELEGRAM_CHAT_ID: ${{ secrets.TELEGRAM_CHAT_ID }}
TELEGRAM_CHAT_ID_E2E: ${{ secrets.TELEGRAM_CHAT_ID_E2E }}
DISCORD_CHANNEL_ID_E2E: ${{ secrets.DISCORD_CHANNEL_ID_E2E }}
SLACK_CHANNEL_ID_E2E: ${{ secrets.SLACK_CHANNEL_ID_E2E }}
run: |
set -euo pipefail
npx tsx tools/e2e/live-vitest-invocation.mts run --test-path test/e2e/live/messaging-providers.test.ts
- name: Upload messaging providers artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
openclaw-plugin-runtime-exdev-release:
needs: generate-matrix
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'openclaw-plugin-runtime-exdev-release') }}
runs-on: ubuntu-latest
permissions:
contents: read
timeout-minutes: 55
env:
E2E_JOB: "1"
E2E_TARGET_ID: "openclaw-plugin-runtime-exdev-release"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev-release
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_SANDBOX_NAME: "e2e-oc-exdev-rel"
OPENSHELL_GATEWAY: "nemoclaw"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false
- *dockerhub-auth
- name: Pre-pull release-matched Docker Hub builder image
shell: bash
run: |
set -euo pipefail
docker pull node:22-trixie-slim@sha256:2d9f5c76c8f4dd36e8f253bee5d828a83a6c09f36188f0b0414325232e0b175d
- name: Remove Docker auth before release-pinned fixture
if: always()
uses: NVIDIA/NemoClaw/.github/actions/docker-auth-cleanup@d5f37099766ca82a4516e7d8f0de117cda197fe3
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@c246409193a31133cab10c8a3589001cc0d59eb3
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Run OpenClaw custom-plugin release baseline live test
run: |
set -euo pipefail
test -n "${DOCKER_CONFIG:-}"
test ! -e "${DOCKER_CONFIG}"
test -z "${DOCKERHUB_USERNAME:-}"
test -z "${DOCKERHUB_TOKEN:-}"
env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN \
npx tsx tools/e2e/live-vitest-invocation.mts run \
--test-path test/e2e/live/openclaw-plugin-runtime-exdev.test.ts \
--selector release-baseline
- name: Upload OpenClaw plugin release baseline artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
# Main-push current-lifecycle coverage for #6108 / #3513 / #3127. This
# proves restart/rebuild persistence and target-side runtime-dependency
# replacement across devices without waiting for the release baseline.
openclaw-plugin-runtime-exdev:
needs: generate-matrix
if: ${{ contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'openclaw-plugin-runtime-exdev') }}
runs-on: ubuntu-latest
permissions:
contents: read
# Two bounded 25-minute onboards plus the 20-minute rebuild and 15-minute
# Vitest buffer need 85 minutes; allow 20 more for setup and teardown.
timeout-minutes: 105
env:
E2E_JOB: "1"
E2E_TARGET_ID: "openclaw-plugin-runtime-exdev"
E2E_ARTIFACT_DIR: ${{ github.workspace }}/e2e-artifacts/live/openclaw-plugin-runtime-exdev
NEMOCLAW_CLI_BIN: ${{ github.workspace }}/bin/nemoclaw.js
NEMOCLAW_RUN_LIVE_E2E: "1"
NEMOCLAW_NON_INTERACTIVE: "1"
NEMOCLAW_ACCEPT_THIRD_PARTY_SOFTWARE: "1"
NEMOCLAW_SANDBOX_NAME: "e2e-oc-exdev"
OPENSHELL_GATEWAY: "nemoclaw"
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false
- *dockerhub-auth
- name: Pre-pull release-matched Docker Hub builder image
shell: bash
run: |
set -euo pipefail
docker pull node:22-trixie-slim@sha256:db8a96a63e5264607ada2d206758876ebbed6a12be2ada7517793cbfb0c2a29c
- name: Remove Docker auth before release-pinned fixture
if: always()
uses: NVIDIA/NemoClaw/.github/actions/docker-auth-cleanup@d5f37099766ca82a4516e7d8f0de117cda197fe3
- name: Prepare E2E workspace
uses: NVIDIA/NemoClaw/.github/actions/prepare-e2e@f6304bc25fc35bfaa441c8c2fbfee38f72805a75
with:
build-cli: "false"
- name: Restore exact-commit CLI artifact
uses: NVIDIA/NemoClaw/.github/actions/restore-e2e-cli-artifact@c246409193a31133cab10c8a3589001cc0d59eb3
with:
provenance-json: ${{ needs.generate-matrix.outputs.cli_artifact_provenance }}
- name: Run OpenClaw custom-plugin lifecycle and runtime-deps EXDEV live test
run: |
set -euo pipefail
test -n "${DOCKER_CONFIG:-}"
test ! -e "${DOCKER_CONFIG}"
test -z "${DOCKERHUB_USERNAME:-}"
test -z "${DOCKERHUB_TOKEN:-}"
env -u DOCKER_CONFIG -u DOCKERHUB_USERNAME -u DOCKERHUB_TOKEN \
npx tsx tools/e2e/live-vitest-invocation.mts run \
--test-path test/e2e/live/openclaw-plugin-runtime-exdev.test.ts \
--selector current-lifecycle
- name: Upload OpenClaw plugin runtime-deps EXDEV artifacts
if: always()
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
- name: Clean up Docker auth
if: always()
shell: bash
run: bash .github/scripts/docker-auth-cleanup.sh
report-to-pr:
runs-on: ubuntu-latest
timeout-minutes: 15
# This entire workflow is dispatch-only. Keeping selective jobs in `needs`
# makes the report wait for and record any requested job without adding
# skipped checks to the normal pull_request workflow.
needs: &e2e-result-jobs
[
base-image-publication,
generate-matrix,
retired-selector-compatibility,
staging-brev-launchable,
live,
shared-e2e,
catalogue-standard,
catalogue-nvidia-api,
catalogue-nvidia-inference,
catalogue-github-read,
catalogue-brave-nvidia-inference,
openshell-gateway-auth-contract,
mcp-bridge,
openshell-credential-generation-window,
openshell-dev-artifact,
mcp-bridge-dev,
managed-image-multiarch-startup,
llama-cpp-dgx-spark-plan,
llama-cpp-dgx-spark-qualification,
managed-image-protected-runtime,
hermes-e2e,
hermes-gpu-startup,
messaging-providers,
jetson-nvmap-gpu,
cloud-onboard,
openclaw-plugin-runtime-exdev-release,
openclaw-plugin-runtime-exdev,
native-runtime-qualification-producer,
native-runtime-qualification-producer-aggregate,
]
if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' }}
permissions:
actions: read
contents: read
# The issue-comment endpoint accepts pull request write permission for PR comments.
# Keep issues: write absent so this job cannot restore general issue routing.
pull-requests: write
steps:
- name: Check out the trusted E2E reporting helper
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.workflow_sha }}
persist-credentials: false
sparse-checkout: |
tools/e2e/report-e2e-results.mts
tools/e2e/selector-aliases.mts
sparse-checkout-cone-mode: false
- name: Post E2E target results to PR
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
EXPLICIT_ONLY_JOBS: ${{ needs.generate-matrix.outputs.explicit_only_jobs }}
TEST_MATRIX: ${{ needs.generate-matrix.outputs.test_matrix }}
JOB_PR_NUMBER: ${{ inputs.pr_number }}
JOB_TARGETS: ${{ inputs.targets }}
JOBS: ${{ inputs.jobs }}
NEEDS_JSON: ${{ toJSON(needs) }}
with:
script: |
const path = require('node:path');
const { pathToFileURL } = require('node:url');
const { resolveReportPr, loadReportJobs, renderE2eReport } = await import(
pathToFileURL(path.join(process.env.GITHUB_WORKSPACE, 'tools/e2e/report-e2e-results.mts')).href
);
const needs = JSON.parse(process.env.NEEDS_JSON || '{}');
const prNumber = await resolveReportPr({ github, context, core, env: process.env });
if (prNumber === undefined) return;
const { apiJobs, loaded } = await loadReportJobs({ github, context, core });
const report = renderE2eReport({
needs,
env: process.env,
apiJobs,
apiJobsLoaded: loaded,
context,
});
if (report.fatal) {
core.setFailed(report.fatal);
return;
}
for (const warning of report.warnings) core.warning(warning);
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
issue_number: prNumber,
body: report.body,
});
relevant-e2e:
name: Relevant E2E
runs-on: ubuntu-latest
timeout-minutes: 5
needs: *e2e-result-jobs
if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'push' }}
permissions:
contents: read
steps:
- name: Check out the E2E result evaluator
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.workflow_sha }}
persist-credentials: false
sparse-checkout: tools/e2e/release-qualification.mts
sparse-checkout-cone-mode: false
- name: Require every selected E2E result
env:
NEEDS_JSON: ${{ toJSON(needs) }}
RELEASE_REQUIRED_JOBS: ${{ needs.generate-matrix.outputs.selected_workflow_jobs }}
run: node --experimental-strip-types --no-warnings tools/e2e/release-qualification.mts
- name: Summarize selected E2E
env:
SELECTED_WORKFLOW_JOBS: ${{ needs.generate-matrix.outputs.selected_workflow_jobs }}
run: |
set -euo pipefail
if [ "${SELECTED_WORKFLOW_JOBS}" = "[]" ]; then
echo "No E2E target owns a file changed by this push." >> "${GITHUB_STEP_SUMMARY}"
else
{
echo "The selected E2E workflow jobs passed:"
echo
jq -r '.[] | "- `\(.)`"' <<< "${SELECTED_WORKFLOW_JOBS}"
} >> "${GITHUB_STEP_SUMMARY}"
fi
release-qualification:
name: Release qualification
runs-on: ubuntu-latest
timeout-minutes: 5
needs: *e2e-result-jobs
if: ${{ always() && github.repository == 'NVIDIA/NemoClaw' && github.ref == 'refs/heads/main' && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' && inputs.jobs == '' && inputs.targets == '' && inputs.include_staging_brev_launchable && !inputs.allow_jetson_dispatch && !inputs.allow_dgx_spark_runner_queue }}
permissions:
contents: read
steps:
- name: Check out the qualification evaluator
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
ref: ${{ github.workflow_sha }}
persist-credentials: false
sparse-checkout: tools/e2e/release-qualification.mts
sparse-checkout-cone-mode: false
- name: Require every release E2E result
env:
NEEDS_JSON: ${{ toJSON(needs) }}
RELEASE_QUALIFICATION_WAIVED_JOBS: ${{ needs.generate-matrix.outputs.release_qualification_waived_jobs }}
RELEASE_REQUIRED_JOBS: ${{ needs.generate-matrix.outputs.release_required_jobs }}
run: node --experimental-strip-types --no-warnings tools/e2e/release-qualification.mts
- name: Record release qualification waiver
if: ${{ inputs.release_qualification_waived_jobs != '' }}
env:
ACTOR: ${{ github.actor }}
CANDIDATE_SHA: ${{ github.sha }}
NEEDS_JSON: ${{ toJSON(needs) }}
RUN_ATTEMPT: ${{ github.run_attempt }}
RUN_ID: ${{ github.run_id }}
TRIGGERING_ACTOR: ${{ github.triggering_actor }}
WAIVED_JOBS: ${{ needs.generate-matrix.outputs.release_qualification_waived_jobs }}
WAIVER_REASON: ${{ inputs.release_qualification_waiver_reason }}
shell: bash
run: |
set -euo pipefail
evidence_dir="${RUNNER_TEMP}/release-qualification-waiver"
install -d -m 0700 "$evidence_dir"
jq -n \
--arg actor "$ACTOR" \
--arg candidateSha "$CANDIDATE_SHA" \
--arg reason "$WAIVER_REASON" \
--arg triggeringActor "$TRIGGERING_ACTOR" \
--argjson needs "$NEEDS_JSON" \
--argjson waived "$WAIVED_JOBS" \
--argjson workflowRunAttempt "$RUN_ATTEMPT" \
--argjson workflowRunId "$RUN_ID" \
'{
schemaVersion: 1,
kind: "nemoclaw-release-qualification-waiver-v1",
candidateSha: $candidateSha,
workflowRunId: $workflowRunId,
workflowRunAttempt: $workflowRunAttempt,
actor: $actor,
triggeringActor: $triggeringActor,
reason: $reason,
jobs: [$waived[] as $job | {id: $job, result: $needs[$job].result}]
}' >"$evidence_dir/waiver.json"
{
echo "## Release qualification waiver"
echo
echo "A repository administrator waived release-required E2E jobs."
echo
printf -- '- Candidate commit: `%s`\n' "$CANDIDATE_SHA"
printf -- '- Dispatch actor: `%s`\n' "$ACTOR"
printf -- '- Triggering actor: `%s`\n' "$TRIGGERING_ACTOR"
printf -- '- Reason: %s\n' "$WAIVER_REASON"
echo '- Waived jobs:'
jq -r --argjson waived "$WAIVED_JOBS" '
. as $needs |
$waived[] | " - `\(.)`: `\($needs[.].result // \"missing\")`"
' <<< "$NEEDS_JSON"
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload release qualification waiver evidence
if: ${{ inputs.release_qualification_waived_jobs != '' }}
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: release-qualification-waiver-${{ github.run_id }}-${{ github.run_attempt }}
path: ${{ runner.temp }}/release-qualification-waiver/waiver.json
if-no-files-found: error
retention-days: 30
# ── Push/manual scorecard ─────────────────────────────────────────────────
scorecard:
runs-on: ubuntu-latest
timeout-minutes: 15
needs: *e2e-result-jobs
if: ${{ always() && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '')) }}
permissions:
actions: read
contents: read
steps:
- name: Checkout scorecard builders
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
repository: ${{ inputs.checkout_repository || github.repository }}
ref: ${{ inputs.checkout_sha || github.sha }}
persist-credentials: false
sparse-checkout: |
ci/onboard-performance-budget.json
scripts/audit-test-runtime.mts
scripts/scorecard
sparse-checkout-cone-mode: false
- name: Download E2E progress artifacts
continue-on-error: true
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: ${{ runner.temp }}/e2e-runtime-audit
pattern: e2e-*
- name: Generate E2E scorecard
id: scorecard
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
EXPLICIT_ONLY_JOBS: ${{ needs.generate-matrix.outputs.explicit_only_jobs }}
JOBS: ${{ inputs.jobs }}
RUNTIME_ARTIFACTS: ${{ runner.temp }}/e2e-runtime-audit
RUNTIME_SUMMARY_FILE: ${{ runner.temp }}/e2e-runtime-summary.json
TARGETS: ${{ inputs.targets }}
NEEDS_JSON: ${{ toJSON(needs) }}
with:
script: |
const path = require('path');
const { buildScorecard } = require(
path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/coordinate-scorecard.mts'),
);
const scorecardJobs = require(
path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/summarize-jobs.mts'),
);
const traceTiming = require(
path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-trace-timing.mts'),
);
const runtimeAudit = require(
path.join(process.env.GITHUB_WORKSPACE, 'scripts/audit-test-runtime.mts'),
);
const runtimeHistory = require(
path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-runtime-history.mts'),
);
const firstTurnLatency = require(
path.join(process.env.GITHUB_WORKSPACE, 'scripts/scorecard/analyze-first-turn-latency.mts'),
);
const needs = JSON.parse(process.env.NEEDS_JSON || '{}');
// GitHub's jobs API is the canonical source because `needs.live`
// collapses every matrix target into one result and has no job URL.
// The typed helper owns and tests the degraded `needs` fallback.
const apiJobs = await scorecardJobs.loadWorkflowRunJobs({ github, context, core });
let runtimeSummaryMarkdown;
let runtimeHistoryRows = null;
try {
const runtimeRows = runtimeAudit.auditTestRuntime([process.env.RUNTIME_ARTIFACTS]);
runtimeSummaryMarkdown = runtimeAudit.formatRuntimeAuditSummary(runtimeRows);
runtimeHistoryRows = runtimeAudit.collectRuntimeHistorySamples([
process.env.RUNTIME_ARTIFACTS,
]);
} catch {
core.warning('E2E test phase runtime summary unavailable: invalid progress artifact');
runtimeSummaryMarkdown = [
'## E2E Test Phase Runtime',
'',
'The summary is unavailable because a `test-progress.json` artifact was invalid.',
'',
].join('\n');
}
const runtimeHistoryMarkdown = runtimeHistoryRows === null
? [
'## E2E Push Runtime Trend',
'',
'The trend is unavailable because a `test-progress.json` artifact was invalid.',
'',
].join('\n')
: await runtimeHistory.buildRuntimeHistory(
{ github, context, core },
runtimeHistoryRows,
process.env.RUNTIME_SUMMARY_FILE,
{
currentFirstTurnLatency:
firstTurnLatency.readCurrentFirstTurnLatencySample(
process.env.RUNTIME_ARTIFACTS,
),
loadPriorPushSummaries: runtimeHistory.loadPriorPushSummaries,
},
);
const trace = await traceTiming.buildTraceTimingResult({ github, context, core });
if (trace.budgetWarningMessage) core.warning(trace.budgetWarningMessage);
const { summaryMarkdown, scorecardData, slackData } = buildScorecard({
eventName: context.eventName,
actor: context.actor || '',
serverUrl: context.serverUrl,
repo: context.repo,
runId: context.runId,
rawJobs: process.env.JOBS || '',
rawTargets: process.env.TARGETS || '',
rawExplicitOnly: process.env.EXPLICIT_ONLY_JOBS || '',
needs,
apiJobs,
trace,
today: new Date().toLocaleDateString('en-US', { month: 'short', day: 'numeric' }),
});
await core.summary
.addRaw(`${summaryMarkdown}\n\n${runtimeSummaryMarkdown}\n${runtimeHistoryMarkdown}`)
.write();
core.setOutput('scorecardData', JSON.stringify(scorecardData));
core.setOutput('slackData', JSON.stringify(slackData));
- name: Post scorecard to Slack
# Webhook secrets never enter branch-dispatched runs. The payload is
# computed in the preceding no-secret step; this fixed publisher does
# not load code from the checked-out workflow ref.
if: ${{ steps.scorecard.outputs.slackData != '' && github.ref == 'refs/heads/main' }}
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
env:
SLACK_WEBHOOK_URL_DAILY: ${{ secrets.SLACK_WEBHOOK_URL_DAILY }}
SLACK_WEBHOOK_URL_FULLRUN: ${{ secrets.SLACK_WEBHOOK_URL_FULLRUN }}
SLACK_WEBHOOK_URL_PREVIEW: ${{ secrets.SLACK_WEBHOOK_URL_PREVIEW }}
SLACK_DATA: ${{ steps.scorecard.outputs.slackData }}
POST_TO_SLACK: ${{ inputs.post_to_slack }}
with:
script: |
const data = JSON.parse(process.env.SLACK_DATA);
const envByChannel = {
daily: 'SLACK_WEBHOOK_URL_DAILY',
fullrun: 'SLACK_WEBHOOK_URL_FULLRUN',
preview: 'SLACK_WEBHOOK_URL_PREVIEW',
};
const channel = data?.channel;
const payload = data?.payload;
const attachments = payload?.attachments;
const validPayload =
Object.hasOwn(envByChannel, channel) &&
payload !== null &&
typeof payload === 'object' &&
typeof payload.text === 'string' &&
Array.isArray(attachments) &&
attachments.every(
(attachment) =>
attachment !== null &&
typeof attachment === 'object' &&
typeof attachment.color === 'string' &&
Array.isArray(attachment.blocks),
);
if (!validPayload) {
core.setFailed('Invalid precomputed Slack payload');
return;
}
if (channel === 'preview' && process.env.POST_TO_SLACK !== 'true') {
core.info('Selective dispatch without post_to_slack — skipping');
return;
}
const webhookUrl = process.env[envByChannel[channel]];
if (!webhookUrl) {
core.info(`Slack webhook for "${channel}" not configured — skipping`);
return;
}
const response = await fetch(webhookUrl, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(data.payload),
});
if (!response.ok) {
core.setFailed(`Slack webhook returned ${response.status}`);
}
- name: Upload E2E runtime summary
if: ${{ always() && github.event_name == 'push' }}
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
with:
name: e2e-runtime-summary
path: ${{ runner.temp }}/e2e-runtime-summary.json