Skip to content

Commit 65aee4f

Browse files
ayuskauskasayuskauskas
committed
test(e2e): add cluster-backed snapshot run-isolation checks
Signed-off-by: Alex Yuskauskas <ayuskauskas@nvidia.com>
1 parent d82dedc commit 65aee4f

1 file changed

Lines changed: 203 additions & 0 deletions

File tree

‎tests/e2e/run.sh‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -467,6 +467,208 @@ test_snapshot() {
467467
fi
468468
}
469469

470+
# =============================================================================
471+
# Snapshot Run Isolation Tests (ADR-020, issue #2120)
472+
# =============================================================================
473+
474+
# Verifies that concurrent snapshot runs own and delete only their own
475+
# Kubernetes resources. These checks are cluster-backed on purpose: the
476+
# unit tests in pkg/k8s/agent run against a fake clientset, which runs no
477+
# Job controller (so pod ownerReferences must be hand-seeded there) and does
478+
# not enforce metav1.Preconditions on delete. Only a real apiserver exercises
479+
# both.
480+
#
481+
# Scope note on UID preconditions: the delete-with-stale-UID race itself is
482+
# not reachable from outside the CLI process — the window between an object's
483+
# creation and the deferred Cleanup is the Job wait, and swapping a live
484+
# object's UID mid-run revokes the running agent's own credentials. That
485+
# mechanism is covered by TestCleanupPassesUIDPrecondition and
486+
# TestCleanupTreatsConflictAsSuccess in pkg/k8s/agent/deployer_test.go. What
487+
# is verified here is the ownership contract those preconditions enforce and
488+
# that IS externally observable: cleanup deletes only objects this run
489+
# created, never one that merely matches its labels or name shape.
490+
test_snapshot_run_isolation() {
491+
msg "=========================================="
492+
msg "Testing snapshot run isolation"
493+
msg "=========================================="
494+
495+
if [ "$FAKE_GPU_ENABLED" != "true" ]; then
496+
skip "snapshot/isolation" "Fake GPU not enabled"
497+
return 0
498+
fi
499+
500+
local ns="$SNAPSHOT_NAMESPACE"
501+
local decoy="aicr-node-reader-e2edecoy"
502+
local rc=0
503+
504+
# --- Decoy: labelled like a run-owned object, but created by nobody's run ---
505+
# A cleanup that swept by label or by name shape would collect this. A
506+
# cleanup scoped to its own created-set must leave it alone.
507+
msg "--- Test: cleanup is scoped to created objects, not to labels ---"
508+
kubectl delete clusterrole "$decoy" --ignore-not-found=true > /dev/null 2>&1 || true
509+
kubectl create clusterrole "$decoy" --verb=get --resource=nodes > /dev/null 2>&1 || true
510+
kubectl label clusterrole "$decoy" \
511+
app.kubernetes.io/name=aicr \
512+
app.kubernetes.io/managed-by=aicr \
513+
app.kubernetes.io/component=snapshot-agent \
514+
aicr.run/run-id=e2edecoy --overwrite > /dev/null 2>&1 || true
515+
local decoy_uid_before
516+
decoy_uid_before=$(kubectl get clusterrole "$decoy" -o jsonpath='{.metadata.uid}' 2>/dev/null || echo "")
517+
518+
# --- Retained run: --no-cleanup, so its objects must outlive later runs ---
519+
msg "--- Test: a retained run's resources survive concurrent runs ---"
520+
local retained_log="${OUTPUT_DIR}/isolation-retained.log"
521+
"${AICR_BIN}" snapshot \
522+
--image "${AICR_IMAGE}" \
523+
--namespace "${ns}" \
524+
--no-cleanup \
525+
--output "${OUTPUT_DIR}/isolation-retained.yaml" \
526+
--timeout 180s \
527+
--privileged \
528+
--node-selector kubernetes.io/os=linux > "$retained_log" 2>&1 || rc=$?
529+
530+
if [ "$rc" -ne 0 ]; then
531+
cat "$retained_log"
532+
fail "snapshot/isolation/retained-run" "retained snapshot run failed"
533+
return 1
534+
fi
535+
536+
local retained_id
537+
retained_id=$(kubectl get jobs -n "$ns" -l app.kubernetes.io/name=aicr \
538+
-o jsonpath='{.items[0].metadata.labels.aicr\.run/run-id}' 2>/dev/null || echo "")
539+
if [ -z "$retained_id" ]; then
540+
fail "snapshot/isolation/run-id-label" "no aicr.run/run-id label on the retained Job"
541+
return 1
542+
fi
543+
detail "retained run ID: ${retained_id}"
544+
pass "snapshot/isolation/run-id-label"
545+
546+
# Every run-owned object must carry the run ID in its name.
547+
local missing=""
548+
kubectl get job -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || missing="${missing} job"
549+
kubectl get sa -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || missing="${missing} sa"
550+
kubectl get role -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || missing="${missing} role"
551+
kubectl get rolebinding -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || missing="${missing} rolebinding"
552+
kubectl get cm -n "$ns" "aicr-agent-snapshot-${retained_id}" > /dev/null 2>&1 || missing="${missing} staging-cm"
553+
kubectl get clusterrole "aicr-node-reader-${retained_id}" > /dev/null 2>&1 || missing="${missing} clusterrole"
554+
kubectl get clusterrolebinding "aicr-node-reader-${retained_id}" > /dev/null 2>&1 || missing="${missing} clusterrolebinding"
555+
if [ -n "$missing" ]; then
556+
fail "snapshot/isolation/run-scoped-names" "not found under run-scoped names:${missing}"
557+
return 1
558+
fi
559+
pass "snapshot/isolation/run-scoped-names"
560+
561+
# The agent's staging ConfigMap must not reuse the validator's name shape.
562+
# aicr validate hands one run ID to both subsystems in one namespace, so a
563+
# shared aicr-snapshot- prefix would give two owners one object.
564+
if kubectl get cm -n "$ns" "aicr-snapshot-${retained_id}" > /dev/null 2>&1; then
565+
fail "snapshot/isolation/staging-cm-name" "found aicr-snapshot-${retained_id}; collides with the validator's data ConfigMap"
566+
return 1
567+
fi
568+
pass "snapshot/isolation/staging-cm-name"
569+
570+
# The pod must be authorized by its controlling ownerReference, not by a
571+
# label — pod labels are writable by anything that can update pods.
572+
msg "--- Test: pod is owned by its Job (controlling ownerReference) ---"
573+
local pod job_uid owner_uid
574+
pod=$(kubectl get pods -n "$ns" -l "aicr.run/run-id=${retained_id}" \
575+
-o jsonpath='{.items[0].metadata.name}' 2>/dev/null || echo "")
576+
job_uid=$(kubectl get job -n "$ns" "aicr-${retained_id}" -o jsonpath='{.metadata.uid}' 2>/dev/null || echo "")
577+
owner_uid=$(kubectl get pod -n "$ns" "$pod" \
578+
-o jsonpath='{.metadata.ownerReferences[?(@.controller==true)].uid}' 2>/dev/null || echo "")
579+
if [ -n "$pod" ] && [ -n "$job_uid" ] && [ "$job_uid" = "$owner_uid" ]; then
580+
detail "pod ${pod} controlled by Job UID ${job_uid}"
581+
pass "snapshot/isolation/pod-owned-by-job"
582+
else
583+
fail "snapshot/isolation/pod-owned-by-job" "pod=${pod} jobUID=${job_uid} ownerUID=${owner_uid}"
584+
return 1
585+
fi
586+
587+
# --- Two concurrent runs, with the retained run's objects still present ---
588+
msg "--- Test: two concurrent runs are independent ---"
589+
local log_a="${OUTPUT_DIR}/isolation-a.log"
590+
local log_b="${OUTPUT_DIR}/isolation-b.log"
591+
local rc_a=0 rc_b=0 pid_a pid_b
592+
593+
"${AICR_BIN}" snapshot --image "${AICR_IMAGE}" --namespace "${ns}" \
594+
--output "${OUTPUT_DIR}/isolation-a.yaml" --timeout 180s --privileged \
595+
--node-selector kubernetes.io/os=linux > "$log_a" 2>&1 &
596+
pid_a=$!
597+
"${AICR_BIN}" snapshot --image "${AICR_IMAGE}" --namespace "${ns}" \
598+
--output "${OUTPUT_DIR}/isolation-b.yaml" --timeout 180s --privileged \
599+
--node-selector kubernetes.io/os=linux > "$log_b" 2>&1 &
600+
pid_b=$!
601+
602+
wait "$pid_a" || rc_a=$?
603+
wait "$pid_b" || rc_b=$?
604+
605+
if [ "$rc_a" -ne 0 ] || [ "$rc_b" -ne 0 ]; then
606+
echo "--- concurrent run A ---"; tail -30 "$log_a"
607+
echo "--- concurrent run B ---"; tail -30 "$log_b"
608+
fail "snapshot/isolation/concurrent-runs" "run A rc=${rc_a}, run B rc=${rc_b}"
609+
return 1
610+
fi
611+
612+
local id_a id_b
613+
id_a=$(grep -o 'runID=[0-9a-f-]*' "$log_a" | head -1 | cut -d= -f2 || echo "")
614+
id_b=$(grep -o 'runID=[0-9a-f-]*' "$log_b" | head -1 | cut -d= -f2 || echo "")
615+
if [ -z "$id_a" ] || [ -z "$id_b" ] || [ "$id_a" = "$id_b" ]; then
616+
fail "snapshot/isolation/concurrent-runs" "expected two distinct run IDs, got '${id_a}' and '${id_b}'"
617+
return 1
618+
fi
619+
detail "run A: ${id_a}"
620+
detail "run B: ${id_b}"
621+
if [ ! -s "${OUTPUT_DIR}/isolation-a.yaml" ] || [ ! -s "${OUTPUT_DIR}/isolation-b.yaml" ]; then
622+
fail "snapshot/isolation/concurrent-runs" "a concurrent run produced an empty snapshot"
623+
return 1
624+
fi
625+
pass "snapshot/isolation/concurrent-runs"
626+
627+
# The retained run's objects must be untouched by both concurrent runs.
628+
# Before ADR-020 this is exactly what broke: a second run's ensureJob
629+
# deleted the same-named Job and its cleanup deleted the shared RBAC.
630+
local destroyed=""
631+
kubectl get job -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} job"
632+
kubectl get sa -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} sa"
633+
kubectl get role -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} role"
634+
kubectl get rolebinding -n "$ns" "aicr-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} rolebinding"
635+
kubectl get clusterrole "aicr-node-reader-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} clusterrole"
636+
kubectl get clusterrolebinding "aicr-node-reader-${retained_id}" > /dev/null 2>&1 || destroyed="${destroyed} clusterrolebinding"
637+
if [ -n "$destroyed" ]; then
638+
fail "snapshot/isolation/retained-run-survives" "concurrent runs destroyed:${destroyed}"
639+
return 1
640+
fi
641+
pass "snapshot/isolation/retained-run-survives"
642+
643+
# Each concurrent run must have removed its own resources.
644+
local leftover_jobs
645+
leftover_jobs=$(kubectl get jobs -n "$ns" -l app.kubernetes.io/name=aicr -o name 2>/dev/null | wc -l | tr -d ' ')
646+
if [ "$leftover_jobs" != "1" ]; then
647+
kubectl get jobs -n "$ns" -l app.kubernetes.io/name=aicr -o name || true
648+
fail "snapshot/isolation/self-cleanup" "expected only the retained Job to remain, found ${leftover_jobs}"
649+
return 1
650+
fi
651+
pass "snapshot/isolation/self-cleanup"
652+
653+
# The decoy must have survived every run above.
654+
local decoy_uid_after
655+
decoy_uid_after=$(kubectl get clusterrole "$decoy" -o jsonpath='{.metadata.uid}' 2>/dev/null || echo "")
656+
if [ -n "$decoy_uid_before" ] && [ "$decoy_uid_before" = "$decoy_uid_after" ]; then
657+
pass "snapshot/isolation/cleanup-scoped-to-created"
658+
else
659+
fail "snapshot/isolation/cleanup-scoped-to-created" \
660+
"aicr-labelled ClusterRole it never created was deleted or replaced (before=${decoy_uid_before} after=${decoy_uid_after})"
661+
return 1
662+
fi
663+
664+
# Housekeeping: remove the decoy and the retained run's resources.
665+
kubectl delete clusterrole "$decoy" --ignore-not-found=true > /dev/null 2>&1 || true
666+
kubectl delete job "aicr-${retained_id}" -n "$ns" --ignore-not-found=true > /dev/null 2>&1 || true
667+
kubectl delete sa,role,rolebinding "aicr-${retained_id}" -n "$ns" --ignore-not-found=true > /dev/null 2>&1 || true
668+
kubectl delete cm "aicr-agent-snapshot-${retained_id}" -n "$ns" --ignore-not-found=true > /dev/null 2>&1 || true
669+
kubectl delete clusterrole,clusterrolebinding "aicr-node-reader-${retained_id}" --ignore-not-found=true > /dev/null 2>&1 || true
670+
}
671+
470672
# =============================================================================
471673
# Recipe from Snapshot Tests (from e2e.md)
472674
# =============================================================================
@@ -1689,6 +1891,7 @@ main() {
16891891
# Setup fake GPU environment and run snapshot tests
16901892
if setup_fake_gpu; then
16911893
test_snapshot
1894+
test_snapshot_run_isolation
16921895
test_recipe_from_snapshot
16931896
test_validate
16941897
test_validate_deployment_checks

0 commit comments

Comments
 (0)