Skip to content

Commit e2afccd

Browse files
ericksoaericksoa
committed
test(e2e): execute native runtime qualification
Signed-off-by: Aaron Erickson <aerickson@nvidia.com>
1 parent 5562e36 commit e2afccd

14 files changed

Lines changed: 3188 additions & 114 deletions

‎.github/workflows/e2e.yaml‎

Lines changed: 122 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1009,18 +1009,14 @@ jobs:
10091009
with:
10101010
node-version: 22.19.0
10111011

1012-
- name: Install locked candidate test dependencies without scripts
1013-
working-directory: .candidate-runtime
1014-
run: npm ci --ignore-scripts
1015-
10161012
- name: Prepare the credential-free execution account and disable Docker
10171013
id: boundary
10181014
env:
10191015
CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime
10201016
shell: bash
10211017
run: |
10221018
set -euo pipefail
1023-
for command in git jq node pgrep podman sha256sum systemctl; do
1019+
for command in git jq node npm pgrep podman sha256sum systemctl; do
10241020
command -v "$command" >/dev/null || {
10251021
echo "::error::Protected runner is missing required command: $command" >&2
10261022
exit 1
@@ -1046,13 +1042,35 @@ jobs:
10461042
runtime_dir="/run/user/${uid}"
10471043
sudo install -d -o "$uid" -g "$uid" -m 0700 "$runtime_dir"
10481044
sudo chown -R "$uid:$uid" "$CANDIDATE_DIRECTORY"
1045+
node_directory="$(dirname "$(command -v node)")"
1046+
[[ "$node_directory" == /* && -x "$node_directory/node" && -x "$node_directory/npm" ]] || {
1047+
echo "::error::Pinned Node toolchain path is invalid" >&2
1048+
exit 1
1049+
}
10491050
guard_dir="${RUNNER_TEMP}/native-runtime-docker-guard"
10501051
install -d -m 0700 "$guard_dir"
10511052
printf '%s\n' '#!/usr/bin/env bash' 'exit 97' >"$guard_dir/docker"
10521053
chmod 0500 "$guard_dir/docker"
10531054
printf 'home=%s\n' "$home" >>"$GITHUB_OUTPUT"
10541055
printf 'runtime_dir=%s\n' "$runtime_dir" >>"$GITHUB_OUTPUT"
10551056
printf 'guard_dir=%s\n' "$guard_dir" >>"$GITHUB_OUTPUT"
1057+
printf 'node_dir=%s\n' "$node_directory" >>"$GITHUB_OUTPUT"
1058+
1059+
- name: Install locked candidate test dependencies without scripts
1060+
env:
1061+
ACCOUNT: ${{ steps.boundary.outputs.account }}
1062+
CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime
1063+
GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }}
1064+
NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }}
1065+
QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }}
1066+
shell: bash
1067+
run: |
1068+
set -euo pipefail
1069+
sudo -u "$ACCOUNT" env -i \
1070+
HOME="$QUALIFICATION_HOME" \
1071+
LANG=C.UTF-8 \
1072+
PATH="$GUARD_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \
1073+
npm --prefix "$CANDIDATE_DIRECTORY" ci --ignore-scripts
10561074
10571075
- name: Run the authenticated installer qualification
10581076
env:
@@ -1084,13 +1102,13 @@ jobs:
10841102
echo "::error::Installer receipt directory is missing or invalid" >&2
10851103
exit 1
10861104
}
1087-
sudo chown -R -h root:root "$INSTALLER_RECEIPT_PARENT/receipts"
10881105
10891106
- name: Execute the candidate qualification case without credentials
10901107
env:
10911108
ACCOUNT: ${{ steps.boundary.outputs.account }}
10921109
CANDIDATE_DIRECTORY: ${{ github.workspace }}/.candidate-runtime
10931110
GUARD_DIRECTORY: ${{ steps.boundary.outputs.guard_dir }}
1111+
NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }}
10941112
QUALIFICATION_HOME: ${{ steps.boundary.outputs.home }}
10951113
RUNTIME_DIRECTORY: ${{ steps.boundary.outputs.runtime_dir }}
10961114
shell: bash
@@ -1113,14 +1131,13 @@ jobs:
11131131
NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_RECEIPT="$receipt_directory/execution.json" \
11141132
NEMOCLAW_NATIVE_RUNTIME_QUALIFICATION_ROW="$QUALIFICATION_ROW" \
11151133
NEMOCLAW_RUN_LIVE_E2E=1 \
1116-
PATH="$GUARD_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \
1134+
PATH="$GUARD_DIRECTORY:$NODE_DIRECTORY:/usr/local/bin:/usr/bin:/bin" \
11171135
XDG_RUNTIME_DIR="$RUNTIME_DIRECTORY" \
11181136
"$CANDIDATE_DIRECTORY/node_modules/.bin/vitest" run \
11191137
--config "$CANDIDATE_DIRECTORY/vitest.config.ts" \
11201138
--project e2e-live \
11211139
"$live_test"
11221140
sudo pkill -KILL -u "$(id -u "$ACCOUNT")" 2>/dev/null || true
1123-
sudo chown -R root:root "$receipt_directory"
11241141
11251142
- name: Verify Docker stayed unavailable
11261143
shell: bash
@@ -1136,14 +1153,14 @@ jobs:
11361153
EVIDENCE_DIRECTORY: ${{ runner.temp }}/native-runtime-evidence
11371154
EXECUTION_RECEIPT_PATH: ${{ runner.temp }}/native-runtime-case/execution.json
11381155
INSTALLER_RECEIPT_DIRECTORY: ${{ runner.temp }}/native-runtime-installer/receipts
1156+
NODE_DIRECTORY: ${{ steps.boundary.outputs.node_dir }}
11391157
shell: bash
11401158
run: |
11411159
set -euo pipefail
1142-
sudo chown -R "$(id -u):$(id -g)" \
1143-
"$INSTALLER_RECEIPT_DIRECTORY" \
1144-
"$(dirname "$EXECUTION_RECEIPT_PATH")"
1145-
node --experimental-strip-types --no-warnings \
1160+
sudo --preserve-env=EVIDENCE_DIRECTORY,EXECUTION_RECEIPT_PATH,INSTALLER_RECEIPT_DIRECTORY,QUALIFICATION_ROW \
1161+
"$NODE_DIRECTORY/node" --experimental-strip-types --no-warnings \
11461162
.trusted-qualification/tools/e2e/native-runtime-qualification-producer-evidence.mts
1163+
sudo chown -R "$(id -u):$(id -g)" "$EVIDENCE_DIRECTORY"
11471164
11481165
- name: Remove qualification resources
11491166
if: always()
@@ -1163,11 +1180,101 @@ jobs:
11631180
fi
11641181
11651182
- name: Upload the qualification case evidence
1166-
if: always()
1183+
if: success()
11671184
uses: NVIDIA/NemoClaw/.github/actions/upload-e2e-artifacts@7768e15eb90d3ee2d33432f481dfe8747e4f6d57
11681185
with:
11691186
name: ${{ matrix.artifactName }}
1170-
path: ${{ runner.temp }}/native-runtime-evidence/evidence.json
1187+
path: ${{ runner.temp }}/native-runtime-evidence/
1188+
1189+
native-runtime-qualification-producer-aggregate:
1190+
name: Aggregate native runtime qualification evidence
1191+
needs:
1192+
[
1193+
generate-matrix,
1194+
native-runtime-qualification-producer-plan,
1195+
native-runtime-qualification-producer,
1196+
]
1197+
if: ${{ always() && needs.native-runtime-qualification-producer-plan.result == 'success' && needs.native-runtime-qualification-producer.result == 'success' && contains(fromJSON(needs.generate-matrix.outputs.selected_jobs), 'native-runtime-qualification-producer') }}
1198+
runs-on: ubuntu-24.04
1199+
timeout-minutes: 15
1200+
permissions:
1201+
actions: read
1202+
contents: read
1203+
pull-requests: read
1204+
steps:
1205+
- name: Check out the trusted qualification aggregator
1206+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
1207+
with:
1208+
ref: ${{ github.workflow_sha }}
1209+
path: .trusted-qualification-aggregate
1210+
persist-credentials: false
1211+
sparse-checkout: |
1212+
src/lib/onboard/runtime-provider/native-qualification-authority.ts
1213+
test/e2e/registry/native-runtime-qualification.ts
1214+
tools/e2e/native-runtime-qualification-producer-aggregate.mts
1215+
sparse-checkout-cone-mode: false
1216+
1217+
- name: Download the exact case evidence cohort
1218+
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
1219+
with:
1220+
pattern: native-runtime-qualification-evidence-${{ inputs.checkout_sha }}-*
1221+
path: ${{ runner.temp }}/native-runtime-case-artifacts
1222+
merge-multiple: false
1223+
1224+
- name: Resolve this aggregate job identity
1225+
id: aggregate-job
1226+
env:
1227+
GH_TOKEN: ${{ github.token }}
1228+
PRODUCER_RUN_ATTEMPT: ${{ github.run_attempt }}
1229+
PRODUCER_RUN_ID: ${{ github.run_id }}
1230+
shell: bash
1231+
run: |
1232+
set -euo pipefail
1233+
jobs="$(gh api --method GET \
1234+
"repos/${GITHUB_REPOSITORY}/actions/runs/${PRODUCER_RUN_ID}/attempts/${PRODUCER_RUN_ATTEMPT}/jobs" \
1235+
-f per_page=100)"
1236+
job_id="$(jq -er \
1237+
--arg name 'Aggregate native runtime qualification evidence' \
1238+
--argjson runId "$PRODUCER_RUN_ID" \
1239+
--argjson attempt "$PRODUCER_RUN_ATTEMPT" '
1240+
select(.total_count <= 100) |
1241+
[.jobs[] | select(
1242+
.name == $name and
1243+
.run_id == $runId and
1244+
.run_attempt == $attempt and
1245+
.status == "in_progress"
1246+
)] |
1247+
select(length == 1) |
1248+
.[0].id
1249+
' <<<"$jobs")" || {
1250+
echo "::error::Could not resolve one in-progress aggregate job identity" >&2
1251+
exit 1
1252+
}
1253+
[[ "$job_id" =~ ^[1-9][0-9]{0,19}$ ]] || {
1254+
echo "::error::Aggregate job identity is invalid" >&2
1255+
exit 1
1256+
}
1257+
printf 'job_id=%s\n' "$job_id" >>"$GITHUB_OUTPUT"
1258+
1259+
- name: Validate and aggregate all 24 case receipts
1260+
working-directory: .trusted-qualification-aggregate
1261+
env:
1262+
AGGREGATE_JOB_ID: ${{ steps.aggregate-job.outputs.job_id }}
1263+
CASE_ARTIFACT_ROOT: ${{ runner.temp }}/native-runtime-case-artifacts
1264+
EVIDENCE_DIRECTORY: ${{ runner.temp }}/native-runtime-aggregate
1265+
QUALIFICATION_PLAN: ${{ needs.native-runtime-qualification-producer-plan.outputs.matrix }}
1266+
run: >-
1267+
node --experimental-strip-types --no-warnings
1268+
tools/e2e/native-runtime-qualification-producer-aggregate.mts
1269+
1270+
- name: Upload the immutable aggregate evidence
1271+
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
1272+
with:
1273+
name: native-runtime-qualification-${{ inputs.checkout_sha }}
1274+
path: ${{ runner.temp }}/native-runtime-aggregate/
1275+
if-no-files-found: error
1276+
retention-days: 30
1277+
compression-level: 9
11711278

11721279
retired-selector-compatibility:
11731280
needs: generate-matrix
@@ -4147,6 +4254,7 @@ jobs:
41474254
openclaw-plugin-runtime-exdev-release,
41484255
openclaw-plugin-runtime-exdev,
41494256
native-runtime-qualification-producer,
4257+
native-runtime-qualification-producer-aggregate,
41504258
]
41514259
if: ${{ always() && github.event_name == 'workflow_dispatch' && inputs.checkout_sha == '' }}
41524260
permissions:

‎scripts/checks/run-native-runtime-installer-qualification.sh‎

Lines changed: 62 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,7 @@ verify_checkout() {
9898
*) fail "$label has an unexpected origin repository." ;;
9999
esac
100100
assert_checkout_has_no_git_credentials "$checkout" "$label"
101+
printf '%s\n' "$revision"
101102
}
102103

103104
verify_committed_file() {
@@ -191,14 +192,51 @@ assert_docker_unavailable() {
191192
[[ ! -S "$socket_path" ]] \
192193
|| fail "A Docker socket exists during the ${phase} check."
193194
done < <(docker_socket_paths)
195+
196+
printf '%s\n' \
197+
'{"dockerCommandGuarded":true,"dockerEnvironmentVariablesUnset":true,"dockerServiceInactive":true,"dockerSocketUnitInactive":true,"dockerdProcessNameAbsent":true,"defaultSocketPathsAbsent":true}'
194198
}
195199

196200
run_native_runtime_installer_qualification() {
197-
candidate_checkout=""
198-
candidate_sha=""
199-
expected_installer_sha256=""
200-
expected_architecture=""
201-
artifact_dir_input=""
201+
local candidate_checkout=""
202+
local candidate_sha=""
203+
local expected_installer_sha256=""
204+
local expected_architecture=""
205+
local artifact_dir_input=""
206+
local artifact_parent=""
207+
local artifact_name=""
208+
local artifact_dir=""
209+
local runner_architecture=""
210+
local candidate_installer=""
211+
local candidate_setup_script=""
212+
local qualification_root=""
213+
local qualification_home=""
214+
local qualification_tmp=""
215+
local docker_guard_dir=""
216+
local managed_payload_root=""
217+
local verified_script_dir=""
218+
local verified_installer=""
219+
local verified_setup_script=""
220+
local installed_checkout=""
221+
local receipt_stage=""
222+
local docker_guard=""
223+
local docker_guard_sha256=""
224+
local candidate_status=0
225+
local verified_candidate_revision=""
226+
local installed_revision=""
227+
local pre_execution_docker_posture=""
228+
local post_execution_docker_posture=""
229+
230+
cleanup() {
231+
if [[ -n "$receipt_stage" && -d "$receipt_stage" && ! -L "$receipt_stage" ]]; then
232+
rm -rf -- "$receipt_stage"
233+
fi
234+
if [[ -n "$qualification_root" && -d "$qualification_root" && ! -L "$qualification_root" ]]; then
235+
rm -rf -- "$qualification_root"
236+
fi
237+
}
238+
trap cleanup EXIT
239+
202240
while [[ "$#" -gt 0 ]]; do
203241
case "$1" in
204242
--candidate-checkout)
@@ -272,7 +310,9 @@ run_native_runtime_installer_qualification() {
272310

273311
candidate_installer="${candidate_checkout}/scripts/install.sh"
274312
candidate_setup_script="${candidate_checkout}/scripts/setup-jetson.sh"
275-
verify_checkout "$candidate_checkout" "$candidate_sha" "The candidate checkout"
313+
verified_candidate_revision="$(
314+
verify_checkout "$candidate_checkout" "$candidate_sha" "The candidate checkout"
315+
)"
276316
verify_installer \
277317
"$candidate_checkout" \
278318
"$candidate_sha" \
@@ -303,16 +343,6 @@ run_native_runtime_installer_qualification() {
303343
"$managed_payload_root" \
304344
"$verified_script_dir"
305345

306-
cleanup() {
307-
if [[ -n "${receipt_stage:-}" && -d "$receipt_stage" && ! -L "$receipt_stage" ]]; then
308-
rm -rf -- "$receipt_stage"
309-
fi
310-
if [[ -n "${qualification_root:-}" && -d "$qualification_root" && ! -L "$qualification_root" ]]; then
311-
rm -rf -- "$qualification_root"
312-
fi
313-
}
314-
trap cleanup EXIT
315-
316346
cp -- "$candidate_installer" "$verified_installer"
317347
cp -- "$candidate_setup_script" "$verified_setup_script"
318348
chmod 500 "$verified_installer" "$verified_setup_script"
@@ -336,9 +366,10 @@ run_native_runtime_installer_qualification() {
336366
PATH="${docker_guard_dir}:${PATH}"
337367
export PATH
338368

339-
assert_docker_unavailable "pre-execution" "$docker_guard" "$docker_guard_sha256"
369+
pre_execution_docker_posture="$(
370+
assert_docker_unavailable "pre-execution" "$docker_guard" "$docker_guard_sha256"
371+
)"
340372

341-
candidate_status=0
342373
# The child shell expands positional parameters inside this literal program.
343374
# shellcheck disable=SC2016
344375
env -i \
@@ -366,11 +397,15 @@ run_native_runtime_installer_qualification() {
366397
install_nemoclaw_before_onboarding
367398
' _ "$verified_installer" "$verified_script_dir" || candidate_status=$?
368399

369-
assert_docker_unavailable "post-execution" "$docker_guard" "$docker_guard_sha256"
400+
post_execution_docker_posture="$(
401+
assert_docker_unavailable "post-execution" "$docker_guard" "$docker_guard_sha256"
402+
)"
370403
[[ "$candidate_status" -eq 0 ]] \
371404
|| fail "The candidate installer phase executor exited with status ${candidate_status}."
372405

373-
verify_checkout "$installed_checkout" "$candidate_sha" "The installed checkout"
406+
installed_revision="$(
407+
verify_checkout "$installed_checkout" "$candidate_sha" "The installed checkout"
408+
)"
374409
verify_installer \
375410
"$installed_checkout" \
376411
"$candidate_sha" \
@@ -382,16 +417,16 @@ run_native_runtime_installer_qualification() {
382417
"$expected_installer_sha256" "$candidate_sha" "$runner_architecture" \
383418
>"${receipt_stage}/invocation.json"
384419
printf '{"receiptVersion":1,"repository":"%s","revision":"%s","installerSha256":"%s"}\n' \
385-
"$CANONICAL_REPOSITORY" "$candidate_sha" "$expected_installer_sha256" \
420+
"$CANONICAL_REPOSITORY" "$verified_candidate_revision" "$expected_installer_sha256" \
386421
>"${receipt_stage}/candidate-source.json"
387422
printf '{"receiptVersion":1,"repository":"%s","requestedRevision":"%s","installedRevision":"%s","installMode":"managed","installerSha256":"%s"}\n' \
388-
"$CANONICAL_REPOSITORY" "$candidate_sha" "$candidate_sha" "$expected_installer_sha256" \
423+
"$CANONICAL_REPOSITORY" "$candidate_sha" "$installed_revision" "$expected_installer_sha256" \
389424
>"${receipt_stage}/installed-source.json"
390425
printf '{"receiptVersion":1,"requested":"%s","runner":"%s"}\n' \
391426
"$expected_architecture" "$runner_architecture" \
392427
>"${receipt_stage}/architecture.json"
393-
printf '%s\n' \
394-
'{"receiptVersion":1,"preExecution":{"dockerCommandGuarded":true,"dockerEnvironmentVariablesUnset":true,"dockerServiceInactive":true,"dockerSocketUnitInactive":true,"dockerdProcessNameAbsent":true,"defaultSocketPathsAbsent":true},"postExecution":{"dockerCommandGuarded":true,"dockerEnvironmentVariablesUnset":true,"dockerServiceInactive":true,"dockerSocketUnitInactive":true,"dockerdProcessNameAbsent":true,"defaultSocketPathsAbsent":true}}' \
428+
printf '{"receiptVersion":1,"preExecution":%s,"postExecution":%s}\n' \
429+
"$pre_execution_docker_posture" "$post_execution_docker_posture" \
395430
>"${receipt_stage}/docker-absence.json"
396431

397432
bounded_file "${receipt_stage}/installer.sh" "$MAX_INSTALLER_BYTES"
@@ -409,6 +444,9 @@ run_native_runtime_installer_qualification() {
409444
receipt_stage=""
410445

411446
printf 'Native runtime installer qualification receipts: %s\n' "$artifact_dir"
447+
cleanup
448+
trap - EXIT
449+
unset -f cleanup
412450
}
413451

414452
if [[ "${BASH_SOURCE[0]:-}" == "$0" ]]; then

‎src/lib/onboard/runtime-provider/native-qualification-authority.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
export const NATIVE_RUNTIME_QUALIFICATION_PROTECTED_REPOSITORY = "NVIDIA/NemoClaw";
1111
/** The trusted collector is separate and rejects evidence emitted by its own workflow. */
1212
export const NATIVE_RUNTIME_QUALIFICATION_PRODUCER_WORKFLOW =
13-
".github/workflows/native-runtime-qualification.yaml";
13+
".github/workflows/e2e.yaml";
1414

1515
export interface NativeRuntimeQualificationProtectedRun {
1616
readonly repository: string;

0 commit comments

Comments
 (0)