refactor(evaluator)!: make runner and agent-eval metrics built-in (#1… #9298

Workflow file for this run

	# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
	# SPDX-License-Identifier: Apache-2.0

	name: CI

	on:
	push:
	branches: [main]
	pull_request:
	branches: [main]
	merge_group:
	types: [checks_requested]
	workflow_dispatch:
	inputs:
	image_tag:
	description: Optional CPU smoke image tag. Defaults to the commit SHA.
	required: false
	type: string
	default: ""
	publish:
	description: Publish CPU smoke images to GHCR.
	required: false
	type: boolean
	default: true

	permissions:
	actions: read
	contents: read
	pull-requests: read

	concurrency:
	group: ${{ github.workflow }}-${{ github.event.pull_request.number \|\| github.ref }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}

	env:
	# Keep the CI uv binary aligned with the Flox development environment.
	UV_VERSION: "0.9.14"
	# Jobs here provision their own toolchain, so Make bypasses Flox activation.
	TOOLCHAIN: system

	jobs:
	changes:
	name: Detect Path Changes
	runs-on: ubuntu-latest
	outputs:
	openapi: ${{ steps.changes.outputs.openapi }}
	test: ${{ steps.changes.outputs.test }}
	deps: ${{ steps.changes.outputs.deps }}
	fabric: ${{ steps.changes.outputs.fabric }}
	evaluator-sdk-closure: ${{ steps.changes.outputs.evaluator-sdk-closure }}
	e2e: ${{ steps.changes.outputs.e2e }}
	docs: ${{ steps.changes.outputs.docs }}
	web-studio: ${{ steps.changes.outputs.web-studio }}
	tools: ${{ steps.changes.outputs.tools }}
	docker: ${{ steps.changes.outputs.docker }}
	helm: ${{ steps.changes.outputs.helm }}
	cpu-smoke: ${{ steps.changes.outputs.cpu-smoke }}
	guardrails-benchmark: ${{ steps.changes.outputs.guardrails-benchmark }}
	deployments-openshell: ${{ steps.changes.outputs.deployments-openshell }}
	ngc-metadata: ${{ steps.changes.outputs.ngc-metadata }}
	auth-idp: ${{ steps.changes.outputs.auth-idp }}
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: ./.github/actions/changes
	id: changes

	actionlint:
	name: Run actionlint
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Install Flox
	uses: flox/install-flox-action@c94e7e1ab56ae14fe98bae4fd84384fd135f0c2a # v2.4.0
	with:
	disable-metrics: "true"
	- name: Lint GitHub workflows and actions
	uses: flox/activate-action@410568008895a0f2e09a34bbd9523f8ef1f2d292 # v1.1.0
	with:
	dir: ./tools/actionlint
	command: actionlint

	ngc-metadata-test:
	name: Test NGC metadata sync
	needs: [changes]
	if: >
	!cancelled() &&
	github.event_name == 'pull_request' &&
	needs.changes.outputs.ngc-metadata == 'true'
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Set up uv
	uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
	with:
	version: ${{ env.UV_VERSION }}
	enable-cache: false
	- name: Run NGC metadata unit tests
	run: uv run --no-cache .github/scripts/tests/test_ngc_metadata.py

	uv-lock:
	name: Check uv lock
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false

	- name: Install platform uv
	uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
	with:
	version: ${{ env.UV_VERSION }}
	enable-cache: false
	python-version: "3.12"

	- name: Show uv version
	run: uv --version

	- name: Check lock with platform uv
	run: bash script/uv-lock.sh --check

	uv-compatibility:
	name: Check configured uv compatibility
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false

	- name: Install configured uv
	uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
	with:
	version: ${{ env.UV_VERSION }}
	enable-cache: false
	python-version: "3.12"

	- name: Show uv version
	run: uv --version

	- name: Check sync with configured uv
	run: uv sync --frozen --all-packages

	docker-bake-graph:
	name: Docker bake graph
	needs: [changes]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.docker == 'true'
	)
	runs-on: ubuntu-latest
	permissions:
	contents: read
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Print bake groups
	shell: bash
	run: \|
	make docker-print TARGET=docker-cpu
	make docker-print TARGET=docker-gpu
	make docker-print TARGET=docker-auditor
	make docker-print TARGET=docker-python-base
	make docker-print TARGET=nmp-automodel-gpu-wheels
	make docker-print TARGET=nmp-automodel
	make docker-print TARGET=nmp-unsloth

	build-cpu-smoke-images:
	name: Build CPU smoke images
	needs: [changes]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.cpu-smoke == 'true' \|\|
	needs.changes.outputs.auth-idp == 'true'
	)
	runs-on: ubuntu-latest
	timeout-minutes: 90
	permissions:
	contents: read
	packages: write
	outputs:
	image_registry: ${{ steps.bake-vars.outputs.image_registry }}
	image_tag: ${{ steps.bake-vars.outputs.image_tag }}
	publish_images: ${{ steps.bake-vars.outputs.publish_images }}
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false

	- name: Free disk space
	uses: ./.github/actions/free-disk-space
	with:
	disable_swap: "true"
	remove_haskell: "true"
	remove_java: "true"
	remove_ruby: "true"
	remove_swift: "true"
	prune_docker: "true"

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
	with:
	name: nmp-builder
	driver: docker-container
	cleanup: true

	- name: Configure bake variables
	id: bake-vars
	shell: bash
	env:
	HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name \|\| github.repository }}
	INPUT_IMAGE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.image_tag \|\| '' }}
	INPUT_PUBLISH: ${{ inputs.publish }}
	SOURCE_SHA: ${{ github.event.pull_request.head.sha \|\| github.sha }}
	run: \|
	set -euo pipefail

	image_registry="ghcr.io/${GITHUB_REPOSITORY,,}"
	source_sha="${SOURCE_SHA:-$GITHUB_SHA}"
	bake_tag="${INPUT_IMAGE_TAG:-$source_sha}"
	fastembed_cache_dir="${RUNNER_TEMP}/fastembed-cache"
	publish_images="false"

	if [ "$GITHUB_EVENT_NAME" = "push" ] && [ "$GITHUB_REF" = "refs/heads/main" ]; then
	publish_images="true"
	fi

	if [ "$GITHUB_EVENT_NAME" = "pull_request" ] && [ "$HEAD_REPOSITORY" = "$GITHUB_REPOSITORY" ]; then
	publish_images="true"
	fi

	if [ "$GITHUB_EVENT_NAME" = "merge_group" ]; then
	publish_images="true"
	fi

	if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ] && [ "$INPUT_PUBLISH" = "true" ]; then
	publish_images="true"
	fi

	{
	printf 'IMAGE_REGISTRY=%s\n' "$image_registry"
	printf 'BASE_REGISTRY=%s\n' "$image_registry"
	printf 'CACHE_REGISTRY=%s\n' "$image_registry"
	printf 'BAKE_TAG=%s\n' "$bake_tag"
	printf 'CI_COMMIT_SHA=%s\n' "$source_sha"
	printf 'FASTEMBED_CACHE_CONTEXT=%s\n' "$fastembed_cache_dir"
	printf 'FASTEMBED_CACHE_DIR=%s\n' "$fastembed_cache_dir"
	printf 'DOCKER_BAKE_ALLOW_FS_READ=%s\n' "$fastembed_cache_dir"
	printf 'FASTEMBED_MODEL_REPO=%s\n' "qdrant/all-MiniLM-L6-v2-onnx"
	printf 'FASTEMBED_MODEL_REVISION=%s\n' "main"
	printf 'PUBLISH_IMAGES=%s\n' "$publish_images"
	} >> "$GITHUB_ENV"
	{
	printf 'image_registry=%s\n' "$image_registry"
	printf 'image_tag=%s\n' "$bake_tag"
	printf 'publish_images=%s\n' "$publish_images"
	} >> "$GITHUB_OUTPUT"

	- name: Prepare fastembed model cache directory
	shell: bash
	run: \|
	set -euo pipefail
	mkdir -p "${FASTEMBED_CACHE_DIR}"
	touch "${FASTEMBED_CACHE_DIR}/.cache-context"

	- name: Restore fastembed model cache
	id: fastembed-cache
	uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
	with:
	path: ${{ runner.temp }}/fastembed-cache
	key: fastembed-qdrant-all-MiniLM-L6-v2-onnx-main-v1

	- name: Install uv for fastembed model cache
	if: steps.fastembed-cache.outputs.cache-hit != 'true'
	uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
	with:
	version: ${{ env.UV_VERSION }}
	cache-dependency-glob: uv.lock

	- name: Populate fastembed model cache
	if: steps.fastembed-cache.outputs.cache-hit != 'true'
	shell: bash
	run: \|
	set -euo pipefail
	HF_HUB_DISABLE_UPDATE_CHECK=1 uv run --no-project --with 'huggingface-hub>=1.0.1,<2.0.0' hf download \
	"${FASTEMBED_MODEL_REPO}" \
	--revision "${FASTEMBED_MODEL_REVISION}" \
	--cache-dir "${FASTEMBED_CACHE_DIR}" \
	--format quiet
	find "${FASTEMBED_CACHE_DIR}/models--qdrant--all-MiniLM-L6-v2-onnx/snapshots" \
	-mindepth 1 -maxdepth 1 -type d -print -quit \| grep -q .

	- name: Log in to GHCR
	if: env.PUBLISH_IMAGES == 'true'
	shell: bash
	env:
	GHCR_TOKEN: ${{ github.token }}
	run: \|
	set -euo pipefail
	echo "$GHCR_TOKEN" \| docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin

	- name: Print Docker bake graph
	shell: bash
	run: \|
	target=docker-cpu
	if [ "${PUBLISH_IMAGES}" = "true" ]; then
	target=docker-cpu-ci
	fi
	make docker-print TARGET="${target}"

	- name: Build CPU images
	if: env.PUBLISH_IMAGES != 'true'
	shell: bash
	run: make docker-load TARGET=docker-cpu

	- name: Build and publish CPU images
	if: env.PUBLISH_IMAGES == 'true'
	shell: bash
	run: make docker-push TARGET=docker-cpu-ci

	- name: Save fastembed model cache
	if: steps.fastembed-cache.outputs.cache-hit != 'true'
	uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
	continue-on-error: true
	with:
	path: ${{ runner.temp }}/fastembed-cache
	key: fastembed-qdrant-all-MiniLM-L6-v2-onnx-main-v1

	build-auditor-test-image:
	name: Build auditor test image
	needs: [changes]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.cpu-smoke == 'true'
	)
	runs-on: ubuntu-latest
	timeout-minutes: 90
	permissions:
	contents: read
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false

	- name: Free disk space
	uses: ./.github/actions/free-disk-space
	with:
	disable_swap: "true"
	remove_haskell: "true"
	remove_java: "true"
	remove_ruby: "true"
	remove_swift: "true"
	prune_docker: "true"

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
	with:
	name: nmp-builder
	driver: docker-container
	cleanup: true

	- name: Configure bake variables
	shell: bash
	env:
	INPUT_IMAGE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.image_tag \|\| '' }}
	SOURCE_SHA: ${{ github.event.pull_request.head.sha \|\| github.sha }}
	run: \|
	set -euo pipefail

	image_registry="ghcr.io/${GITHUB_REPOSITORY,,}"
	source_sha="${SOURCE_SHA:-$GITHUB_SHA}"
	bake_tag="${INPUT_IMAGE_TAG:-$source_sha}"

	{
	printf 'IMAGE_REGISTRY=%s\n' "$image_registry"
	printf 'BASE_REGISTRY=%s\n' "$image_registry"
	printf 'CACHE_REGISTRY=%s\n' "$image_registry"
	printf 'BAKE_TAG=%s\n' "$bake_tag"
	printf 'CI_COMMIT_SHA=%s\n' "$source_sha"
	} >> "$GITHUB_ENV"

	- name: Print Docker bake graph
	shell: bash
	run: make docker-print TARGET=docker-auditor

	- name: Build auditor image
	shell: bash
	run: make docker-build TARGET=docker-auditor

	build-safe-synthesizer-test-image:
	name: Build Safe Synthesizer test image
	needs: [changes]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.cpu-smoke == 'true'
	)
	runs-on: ubuntu-latest
	timeout-minutes: 120
	permissions:
	contents: read
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false

	- name: Free disk space
	uses: ./.github/actions/free-disk-space
	with:
	disable_swap: "true"
	remove_go: "true"
	remove_haskell: "true"
	remove_java: "true"
	remove_node: "true"
	remove_pypy: "true"
	remove_python: "true"
	remove_ruby: "true"
	remove_swift: "true"
	prune_docker: "true"

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # v4.2.0
	with:
	name: nmp-builder
	driver: docker-container
	cleanup: true

	- name: Configure bake variables
	shell: bash
	env:
	INPUT_IMAGE_TAG: ${{ github.event_name == 'workflow_dispatch' && inputs.image_tag \|\| '' }}
	SOURCE_SHA: ${{ github.event.pull_request.head.sha \|\| github.sha }}
	run: \|
	set -euo pipefail

	image_registry="ghcr.io/${GITHUB_REPOSITORY,,}"
	source_sha="${SOURCE_SHA:-$GITHUB_SHA}"
	bake_tag="${INPUT_IMAGE_TAG:-$source_sha}"

	{
	printf 'IMAGE_REGISTRY=%s\n' "$image_registry"
	printf 'BASE_REGISTRY=%s\n' "$image_registry"
	printf 'CACHE_REGISTRY=%s\n' "$image_registry"
	printf 'BAKE_TAG=%s\n' "$bake_tag"
	printf 'CI_COMMIT_SHA=%s\n' "$source_sha"
	} >> "$GITHUB_ENV"

	- name: Print Docker bake graph
	shell: bash
	run: make docker-print TARGET=docker-gpu

	- name: Build Safe Synthesizer image
	shell: bash
	run: make docker-build TARGET=docker-gpu

	kind-cpu-smoke:
	name: Kind CPU smoke test
	needs: [changes, build-cpu-smoke-images]
	if: >
	!cancelled() &&
	needs.build-cpu-smoke-images.result == 'success' &&
	needs.build-cpu-smoke-images.outputs.publish_images == 'true'
	runs-on: ubuntu-latest
	timeout-minutes: 45
	permissions:
	contents: read
	packages: read
	env:
	KIND_CLUSTER_NAME: gha-${{ github.run_id }}-${{ github.run_attempt }}-kind-smoke
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false

	- name: Setup Kind cluster with NeMo Platform
	uses: ./.github/actions/setup-kind-cluster
	with:
	kind-cluster-name: ${{ env.KIND_CLUSTER_NAME }}
	image-registry: ${{ needs.build-cpu-smoke-images.outputs.image_registry }}
	image-tag: ${{ needs.build-cpu-smoke-images.outputs.image_tag }}
	kind-image-pull-token: ${{ github.token }}
	kind-image-pull-user: ${{ github.actor }}

	- name: Run CPU job e2e smoke test
	shell: bash
	env:
	_TYPER_FORCE_DISABLE_TERMINAL: "1"
	E2E_SERVICES_LOG_DIR: ${{ runner.temp }}/e2e-services-logs
	NGC_API_KEY: not-used-for-ghcr-cpu-smoke
	run: \|
	test -n "${NMP_E2E_CLUSTER_URL}"
	export NMP_BASE_URL="${NMP_E2E_CLUSTER_URL}"
	uv run --frozen pytest \
	e2e/test_jobs.py::test_job_using_secret_environment_variable \
	e2e/test_models_deployments_plugin.py::test_generic_model_deployment_lifecycle \
	-v \
	--run-e2e \
	--no-cov \
	--junitxml=report-kubernetes-smoke.xml

	- name: Finalize Kind e2e
	if: always()
	uses: ./.github/actions/finalize-kind-e2e
	with:
	kind-cluster-name: ${{ env.KIND_CLUSTER_NAME }}
	artifact-name: kind-smoke-kubernetes-artifacts
	junit-report: report-kubernetes-smoke.xml
	services-log-dir: ${{ runner.temp }}/e2e-services-logs

	kind-cpu-e2e:
	name: Kind CPU e2e
	needs: [changes, build-cpu-smoke-images]
	if: >
	!cancelled() &&
	needs.build-cpu-smoke-images.result == 'success' &&
	needs.build-cpu-smoke-images.outputs.publish_images == 'true'
	runs-on: ubuntu-latest
	timeout-minutes: 45
	permissions:
	contents: read
	packages: read
	env:
	KIND_CLUSTER_NAME: gha-${{ github.run_id }}-${{ github.run_attempt }}-kind-e2e
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false

	- name: Setup Kind cluster with NeMo Platform
	uses: ./.github/actions/setup-kind-cluster
	with:
	kind-cluster-name: ${{ env.KIND_CLUSTER_NAME }}
	image-registry: ${{ needs.build-cpu-smoke-images.outputs.image_registry }}
	image-tag: ${{ needs.build-cpu-smoke-images.outputs.image_tag }}
	kind-image-pull-token: ${{ github.token }}
	kind-image-pull-user: ${{ github.actor }}

	- name: Run jobs and data-designer e2e tests
	shell: bash
	env:
	_TYPER_FORCE_DISABLE_TERMINAL: "1"
	E2E_SERVICES_LOG_DIR: ${{ runner.temp }}/e2e-services-logs
	NGC_API_KEY: not-used-for-ghcr-cpu-e2e
	# Let the k8s-mode agent deployment e2e (test_nemo_agents_k8s.py) learn
	# the agent image ref. The Helm platform is configured with a k8s
	# deployments executor (e2e/k8s/values/kind.yaml) and this same
	# nmp-api image is pre-pulled into the kind nodes, so the agent pod
	# resolves it node-locally under IfNotPresent.
	NMP_E2E_IMAGE_REGISTRY: ${{ needs.build-cpu-smoke-images.outputs.image_registry }}
	NMP_E2E_IMAGE_TAG: ${{ needs.build-cpu-smoke-images.outputs.image_tag }}
	run: \|
	test -n "${NMP_E2E_CLUSTER_URL}"
	export NMP_BASE_URL="${NMP_E2E_CLUSTER_URL}"
	uv run --frozen pytest \
	e2e \
	-v \
	-m "not gym_e2e" \
	--run-e2e \
	--no-cov \
	--junitxml=report-kubernetes-e2e.xml

	- name: Finalize Kind e2e
	if: always()
	uses: ./.github/actions/finalize-kind-e2e
	with:
	kind-cluster-name: ${{ env.KIND_CLUSTER_NAME }}
	artifact-name: kind-e2e-kubernetes-artifacts
	junit-report: report-kubernetes-e2e.xml
	services-log-dir: ${{ runner.temp }}/e2e-services-logs

	kind-gym-e2e:
	name: Kind Gym agent-evaluate e2e
	needs: [changes, build-cpu-smoke-images]
	if: >
	!cancelled() &&
	needs.build-cpu-smoke-images.result == 'success' &&
	needs.build-cpu-smoke-images.outputs.publish_images == 'true'
	runs-on: ubuntu-latest
	timeout-minutes: 45
	permissions:
	contents: read
	packages: read
	env:
	KIND_CLUSTER_NAME: gha-${{ github.run_id }}-${{ github.run_attempt }}-kind-gym-e2e
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false

	- name: Setup Kind cluster for Gym evaluation
	uses: ./.github/actions/setup-kind-cluster
	with:
	kind-cluster-name: ${{ env.KIND_CLUSTER_NAME }}
	image-registry: ${{ needs.build-cpu-smoke-images.outputs.image_registry }}
	image-tag: ${{ needs.build-cpu-smoke-images.outputs.image_tag }}
	kind-image-pull-token: ${{ github.token }}
	kind-image-pull-user: ${{ github.actor }}

	- name: Pre-pull Gym task image
	shell: bash
	env:
	KIND_IMAGE_PULL_TOKEN: ${{ github.token }}
	KIND_IMAGE_PULL_USER: ${{ github.actor }}
	IMAGE_REGISTRY: ${{ needs.build-cpu-smoke-images.outputs.image_registry }}
	IMAGE_TAG: ${{ needs.build-cpu-smoke-images.outputs.image_tag }}
	run: e2e/k8s/scripts/prepull_kind_images.sh "${IMAGE_REGISTRY}/nmp-gym-tasks:${IMAGE_TAG}"

	- name: Run Gym agent-evaluate e2e test
	shell: bash
	env:
	_TYPER_FORCE_DISABLE_TERMINAL: "1"
	E2E_SERVICES_LOG_DIR: ${{ runner.temp }}/e2e-services-logs
	run: \|
	test -n "${NMP_E2E_CLUSTER_URL}"
	export NMP_BASE_URL="${NMP_E2E_CLUSTER_URL}"
	uv run --frozen pytest \
	e2e/test_evaluator_plugin.py \
	-v \
	-m gym_e2e \
	--run-e2e \
	--no-cov \
	--junitxml=report-kubernetes-gym-e2e.xml

	- name: Finalize Kind Gym e2e
	if: always()
	uses: ./.github/actions/finalize-kind-e2e
	with:
	kind-cluster-name: ${{ env.KIND_CLUSTER_NAME }}
	artifact-name: kind-gym-e2e-kubernetes-artifacts
	junit-report: report-kubernetes-gym-e2e.xml
	services-log-dir: ${{ runner.temp }}/e2e-services-logs

	kind-network-policy-smoke:
	name: Kind NetworkPolicy smoke
	needs: [changes, build-cpu-smoke-images]
	if: >
	!cancelled() &&
	needs.build-cpu-smoke-images.result == 'success' &&
	needs.build-cpu-smoke-images.outputs.publish_images == 'true' && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.helm == 'true' \|\|
	needs.changes.outputs.e2e == 'true'
	)
	runs-on: ubuntu-latest
	timeout-minutes: 45
	permissions:
	contents: read
	packages: read
	env:
	KIND_CLUSTER_NAME: gha-${{ github.run_id }}-${{ github.run_attempt }}-network-policy
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false

	- name: Setup Kind cluster with NetworkPolicy enforcement
	uses: ./.github/actions/setup-kind-cluster
	with:
	kind-cluster-name: ${{ env.KIND_CLUSTER_NAME }}
	image-registry: ${{ needs.build-cpu-smoke-images.outputs.image_registry }}
	image-tag: ${{ needs.build-cpu-smoke-images.outputs.image_tag }}
	kind-image-pull-token: ${{ github.token }}
	kind-image-pull-user: ${{ github.actor }}
	kind-enable-gateway: "false"
	kind-enable-network-policies: "true"
	helm-extra-args: -f e2e/k8s/values/network-policies.yaml
	wait-for-api: "false"

	- name: Run NetworkPolicy smoke test
	shell: bash
	env:
	NAMESPACE: nemo-platform
	CHAINSAW_REPORT_FORMAT: XML
	CHAINSAW_REPORT_NAME: report-network-policy-smoke
	CHAINSAW_REPORT_PATH: .
	run: \|
	if e2e/k8s/scripts/test_network_policies.sh; then
	if [ ! -f report-network-policy-smoke.xml ]; then
	printf '<testsuite name="network-policy-smoke" tests="1" failures="0"/>%s' $'\n' > report-network-policy-smoke.xml
	fi
	exit 0
	fi
	status="$?"
	if [ ! -f report-network-policy-smoke.xml ]; then
	cat > report-network-policy-smoke.xml <<'EOF'
	<testsuite name="network-policy-smoke" tests="1" failures="1">
	<testcase name="network-policy-smoke">
	<failure message="network policy smoke test failed">See the workflow log and Kubernetes artifact bundle.</failure>
	</testcase>
	</testsuite>
	EOF
	fi
	exit "${status}"

	- name: Finalize Kind e2e
	if: always()
	uses: ./.github/actions/finalize-kind-e2e
	with:
	kind-cluster-name: ${{ env.KIND_CLUSTER_NAME }}
	artifact-name: kind-network-policy-smoke-artifacts
	junit-report: report-network-policy-smoke.xml
	services-log-dir: ${{ runner.temp }}/e2e-services-logs

	helm-lint:
	name: Helm lint
	needs: [changes]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.helm == 'true'
	)
	runs-on: ubuntu-latest
	permissions:
	contents: read
	env:
	HELM_FOLDER: k8s/helm
	HELM_RELEASE_NAME: nemo-platform
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Install Flox
	uses: flox/install-flox-action@c94e7e1ab56ae14fe98bae4fd84384fd135f0c2a # v2.4.0
	with:
	disable-metrics: "true"
	- name: Lint and validate Helm chart
	uses: flox/activate-action@410568008895a0f2e09a34bbd9523f8ef1f2d292 # v1.1.0
	with:
	dir: ./tools/helm
	command: tools/lint/lint-helm.sh
	- name: Upload Helm lint artifacts
	if: failure()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: helm-lint-artifacts
	retention-days: 7
	if-no-files-found: ignore
	path: \|
	k8s/helm/ci/*.output
	k8s/helm/ci/*.kubeconform.json

	helm-chart-verifier:
	name: Helm chart verifier
	needs: [changes]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.helm == 'true'
	)
	runs-on: ubuntu-latest
	permissions:
	contents: read
	env:
	CHART_VERIFIER_VERSION: "1.14.0"
	HELM_FOLDER: k8s/helm
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Install Helm
	uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0
	- name: Install chart-verifier
	id: install-chart-verifier
	shell: bash
	env:
	GITHUB_TOKEN: ${{ github.token }}
	run: \|
	set -euo pipefail

	mkdir -p "${RUNNER_TEMP}/chart-verifier"
	chart_verifier_archive="chart-verifier-${CHART_VERIFIER_VERSION}.tgz"
	chart_verifier_archive_path="${RUNNER_TEMP}/chart-verifier.tar.gz"
	release_metadata_path="${RUNNER_TEMP}/chart-verifier-release.json"
	release_url="https://github.com/redhat-certification/chart-verifier/releases/download/${CHART_VERIFIER_VERSION}/${chart_verifier_archive}"
	release_api_url="https://api.github.com/repos/redhat-certification/chart-verifier/releases/tags/${CHART_VERIFIER_VERSION}"

	curl_args=(
	--fail
	--show-error
	--silent
	--location
	--retry 5
	--retry-delay 2
	--retry-all-errors
	)
	github_api_args=(
	"${curl_args[@]}"
	--header "Accept: application/vnd.github+json"
	--header "X-GitHub-Api-Version: 2022-11-28"
	)
	if [ -n "${GITHUB_TOKEN:-}" ]; then
	github_api_args+=(--header "Authorization: Bearer ${GITHUB_TOKEN}")
	fi

	if ! curl "${curl_args[@]}" -o "${chart_verifier_archive_path}" "${release_url}"; then
	echo "::warning::Unable to download chart-verifier ${CHART_VERIFIER_VERSION}; skipping chart verifier."
	echo "installed=false" >> "${GITHUB_OUTPUT}"
	exit 1
	fi

	if ! curl "${github_api_args[@]}" -o "${release_metadata_path}" "${release_api_url}"; then
	echo "::warning::Unable to fetch chart-verifier release metadata; skipping chart verifier."
	echo "installed=false" >> "${GITHUB_OUTPUT}"
	exit 1
	fi

	if ! chart_verifier_sha256="$(
	python3 - "${release_metadata_path}" "${chart_verifier_archive}" <<'PY'
	import json
	import sys

	metadata_path = sys.argv[1]
	target = sys.argv[2]

	with open(metadata_path, encoding="utf-8") as metadata_file:
	release = json.load(metadata_file)

	for asset in release.get("assets", []):
	if asset.get("name") != target:
	continue
	digest = asset.get("digest", "")
	if digest.startswith("sha256:"):
	print(digest.removeprefix("sha256:"))
	sys.exit(0)

	print(f"sha256 digest not found for release asset {target}", file=sys.stderr)
	sys.exit(1)
	PY
	)"; then
	echo "::warning::Unable to read chart-verifier release digest; skipping chart verifier."
	echo "installed=false" >> "${GITHUB_OUTPUT}"
	exit 1
	fi

	printf '%s %s\n' "${chart_verifier_sha256}" "${RUNNER_TEMP}/chart-verifier.tar.gz" \
	\| sha256sum -c -
	tar -xzf "${RUNNER_TEMP}/chart-verifier.tar.gz" -C "${RUNNER_TEMP}/chart-verifier" chart-verifier
	chmod +x "${RUNNER_TEMP}/chart-verifier/chart-verifier"
	echo "${RUNNER_TEMP}/chart-verifier" >> "${GITHUB_PATH}"
	echo "installed=true" >> "${GITHUB_OUTPUT}"
	- name: Prepare Helm dependencies
	if: steps.install-chart-verifier.outputs.installed == 'true'
	shell: bash
	run: \|
	helm repo add nvidia https://helm.ngc.nvidia.com/nvidia
	helm repo update
	helm dep update "${HELM_FOLDER}"
	- name: Run chart verifier
	if: steps.install-chart-verifier.outputs.installed == 'true'
	shell: bash
	run: \|
	set -euo pipefail

	report_json="${RUNNER_TEMP}/chart-verifier-report.json"
	chart-verifier verify --enable helm-lint,is-helm-v3,contains-values,has-readme \
	-o json "${HELM_FOLDER}" \| tee "${report_json}"

	python3 - "${report_json}" <<'PY'
	import json
	import sys

	with open(sys.argv[1], encoding="utf-8") as report_file:
	data = json.load(report_file)

	failed = [
	result.get("check", "?")
	for result in data.get("results", [])
	if result.get("outcome") == "FAIL"
	]
	if failed:
	print("chart-verifier failed checks: " + ", ".join(failed), file=sys.stderr)
	sys.exit(1)
	PY

	lint:
	name: Lint all
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Install flox
	uses: flox/install-flox-action@c94e7e1ab56ae14fe98bae4fd84384fd135f0c2a # v2.4.0
	with:
	disable-metrics: "true"
	- name: Install uv
	uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
	with:
	version: ${{ env.UV_VERSION }}
	enable-cache: true
	python-version: "3.12"
	cache-dependency-glob: uv.lock
	- name: Install nodejs
	uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
	with:
	node-version-file: .nvmrc
	- name: Install pnpm via corepack
	run: \|
	npm i -g corepack@0.31.0
	corepack enable pnpm
	- name: Install web dependencies
	working-directory: web
	run: pnpm install --frozen-lockfile
	- name: Run Lint
	uses: flox/activate-action@410568008895a0f2e09a34bbd9523f8ef1f2d292 # v1.1.0
	env:
	CI_PROJECT_DIR: ${{ github.workspace }}
	with:
	dir: ./tools/lint
	command: \|
	tools/lint/lint-all.sh
	- name: Upload lint artifacts
	if: failure()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: Lint Artifacts
	retention-days: 7
	path: \|
	third_party/
	openapi/openapi.yaml
	openapi/ga/openapi.yaml
	openapicheck/openapi.yaml
	openapicheck/openapi.ga.yaml
	docs/set-up/config-reference.md
	python-sdk-lint/
	web/packages/sdk/generated/
	diff.txt

	policy-wasm:
	name: Build OPA policy WASM
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: ./.github/actions/build-policy-wasm
	- name: Upload policy WASM
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: policy-wasm
	retention-days: 1
	path: services/core/auth/src/nmp/core/auth/assets/policy.wasm

	python-unit-test-tools:
	name: Python unit tests (tools)
	needs: [changes]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.tools == 'true'
	)
	runs-on: ubuntu-latest
	permissions:
	contents: read
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Install uv
	uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
	with:
	version: ${{ env.UV_VERSION }}
	python-version: "3.12"
	enable-cache: true
	cache-dependency-glob: uv.lock
	- name: Run tools unit tests
	run: uv run pytest tools -v --junit-xml=report.xml
	- name: Upload test artifacts
	if: always()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: python-unit-test-tools-results
	retention-days: 30
	path: report.xml

	python-unit-test:
	name: Python unit tests
	needs: [policy-wasm]
	runs-on: ubuntu-latest
	permissions:
	contents: read
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Download policy WASM
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: policy-wasm
	path: services/core/auth/src/nmp/core/auth/assets
	- name: Install uv
	uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
	with:
	version: ${{ env.UV_VERSION }}
	python-version: "3.12"
	enable-cache: true
	cache-dependency-glob: uv.lock
	- name: Run unit tests
	run: make test-unit-ci
	env:
	EXTRA: cpu
	HF_TOKEN: ${{ secrets.HF_TOKEN }}
	PYTEST_WORKERS: "4"
	_TYPER_FORCE_DISABLE_TERMINAL: "1"
	- name: Upload test artifacts
	if: always()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: python-unit-test-results
	retention-days: 30
	path: \|
	report.xml
	coverage.xml
	coverage.json
	coverage-html/

	# WHY THIS JOB EXISTS (it looks redundant, it isn't): `--extra fabric` appears nowhere else in this
	# file and there is no `--all-extras`, so no other job installs nemo-fabric. Every other Fabric test
	# either fakes `nemo_fabric` or `importorskip`s it, which means the hermetic suite keeps passing when
	# the real API moves — that is how the enable_relay keyword, the `.cli`/`.sdk` adapter ids, and the
	# entire profile mechanism each drifted underneath us while the unit tests stayed green. Deleting
	# this job takes real-Fabric coverage to zero. The bare import is also the ONLY check on runtime.py's
	# TYPE_CHECKING import block (RunResult/RunOutput/...), which ty cannot verify because nemo-fabric is
	# absent from the type-check environment.
	fabric-wheel-smoke:
	name: Fabric wheel install smoke (Linux, py${{ matrix.python-version }})
	needs: [changes]
	# A Fabric wheel bump lands in uv.lock (deps), and edits to the runtime's own Fabric call sites or
	# its tests land under the fabric filter — either can surface API drift, so run on both.
	if: >
	!cancelled() &&
	(needs.changes.outputs.deps == 'true' \|\| needs.changes.outputs.fabric == 'true')
	runs-on: ubuntu-latest
	permissions:
	contents: read
	strategy:
	fail-fast: false
	matrix:
	# One interpreter on purpose. nemo-fabric-runtime ships a single cp311-abi3 wheel and the
	# adapters are pure Python, so extra legs mostly re-check the same dependency closure; the
	# 3.12-3.13 range is already asserted by uv.lock resolving universally.
	#
	# It must satisfy uv.lock's `requires-python` (currently >=3.12,<3.14) because this job runs
	# `uv sync` on the WORKSPACE. That ceiling is the intersection across workspace members —
	# nemo-rl, automodel, unsloth, deployments and experimentalist still cap at <3.14 — so it is
	# narrower than the root pyproject's <3.15, and a 3.14 leg would fail at `uv sync` before
	# installing anything.
	python-version: ["3.12"]
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	# This job installs and imports third-party wheels; don't leave GITHUB_TOKEN in .git/config.
	persist-credentials: false
	- name: Install uv
	uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
	with:
	version: ${{ env.UV_VERSION }}
	# Must satisfy the workspace floor in the root pyproject.toml (`requires-python`), or
	# `uv sync` refuses the interpreter outright.
	python-version: ${{ matrix.python-version }}
	enable-cache: true
	cache-dependency-glob: uv.lock
	# Installs the published nemo-fabric wheels (+ codex/claude/deepagents adapters) from the lock.
	# Linux is where jobs actually execute the runtime, and it validates that the manylinux
	# nemo-fabric-runtime wheel installs on the runner's glibc.
	- name: Install nemo-evaluator-sdk[fabric] from the lock
	run: uv sync --frozen --package nemo-evaluator-sdk --extra fabric
	# Import the Fabric SDK surface that agent_eval/runtimes/fabric/runtime.py depends on, so the
	# published package is exercised for real instead of only via the hermetic fake-nemo_fabric tests.
	- name: Import the Fabric SDK surface the runtime uses
	run: \|
	uv run --frozen --no-sync python - <<'PY'
	import nemo_fabric
	from nemo_fabric import (
	EnvironmentConfig,
	Fabric,
	FabricConfig,
	RelayAtifConfig,
	RelayAtofConfig,
	RelayAtofFileSinkConfig,
	RelayObservabilityConfig,
	RunRequest,
	RunResult,
	)
	print("nemo_fabric import OK:", nemo_fabric.__file__)
	PY
	# A bare import can't catch API drift in the runtime's call sites — the enable_relay keyword and
	# the harness adapter ids both moved under us while the hermetic fakes kept passing. These contract
	# tests exercise those call sites against the real wheels (importorskip elsewhere), so a future
	# drift turns this job red instead of slipping through.
	- name: Run Fabric runtime contract tests against the real wheels
	# pytest lives in the root `dev` group, which the minimal `--package ... --extra fabric` sync
	# above does not pull, so install it (and pytest-asyncio, for the SDK package's
	# asyncio_mode=auto) into the synced venv rather than bloating the smoke env with the whole
	# dev group.
	#
	# It must go INTO .venv — NOT via `uv run --with`, which layers an ephemeral overlay whose
	# sys.prefix is a temp build dir. Fabric discovers adapter descriptors under
	# `<sys.prefix>/share/nemo-fabric/adapters` (that is where the adapter wheels install their
	# fabric-adapter.json data files), so under an overlay every adapter goes missing and any
	# resolution fails with `unknown adapter ...; available adapters: []`.
	run: \|
	uv pip install --python .venv/bin/python pytest pytest-asyncio
	uv run --frozen --no-sync pytest \
	packages/nemo_evaluator_sdk/tests/agent_eval/test_fabric_surface.py -v

	# Installs only nemo-evaluator-sdk (no workspace/extras/dev) and imports
	# agent_eval.runtimes.harbor_runtime. Catches undeclared deps (workspace sync hides them)
	# and broken lazy barrels that pull in the metric/execution stack.
	evaluator-sdk-closure-smoke:
	name: Evaluator SDK dependency-closure smoke (Linux, py${{ matrix.python-version }})
	needs: [changes]
	if: >
	!cancelled() &&
	(needs.changes.outputs.deps == 'true' \|\| needs.changes.outputs.evaluator-sdk-closure == 'true')
	runs-on: ubuntu-latest
	permissions:
	contents: read
	strategy:
	fail-fast: false
	matrix:
	# Must satisfy uv.lock's requires-python (>=3.12,<3.14), which this job syncs against.
	python-version: ["3.12"]
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Install uv
	uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
	with:
	python-version: ${{ matrix.python-version }}
	enable-cache: true
	cache-dependency-glob: uv.lock
	- name: Install nemo-evaluator-sdk with no extras and no dev group
	run: uv sync --frozen --package nemo-evaluator-sdk --no-dev
	# pytest is not in the package-only sync; install the runner only (do not add SDK deps).
	# WHY BOTH THIS JOB AND THE UNIT TEST:
	# - Unit-test CI syncs the whole workspace: a sibling (e.g. nemo-platform-sdk) may already
	# provide httpx, so an undeclared SDK dep still imports — green here, broken for consumers
	# who only installed nemo-evaluator-sdk.
	# - This job: `uv sync --package nemo-evaluator-sdk --no-dev` (no extras/siblings) so that
	# undeclared-dep case fails, then runs test_lazy_public_api (laziness assertions; one source
	# of truth — no duplicated heredoc probe).
	- name: Run lazy-import closure unit test under the stripped install
	run: \|
	uv pip install --python .venv/bin/python pytest
	uv run --frozen --no-sync pytest \
	packages/nemo_evaluator_sdk/tests/test_lazy_public_api.py::test_agent_eval_import_does_not_pull_the_execution_stack -v

	python-integration-test:
	name: Python integration tests
	needs: [policy-wasm]
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Free disk space
	uses: ./.github/actions/free-disk-space
	- name: Download policy WASM
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: policy-wasm
	path: services/core/auth/src/nmp/core/auth/assets
	- name: Install uv
	uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
	with:
	version: ${{ env.UV_VERSION }}
	python-version: "3.12"
	enable-cache: true
	cache-dependency-glob: uv.lock
	- name: Log in to Docker Hub
	uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
	with:
	username: ${{ vars.AIRE_DOCKERHUB_USERNAME }}
	password: ${{ secrets.AIRE_DOCKERHUB_ACCESS_TOKEN }}
	- name: Pull mock base images
	run: \|
	docker pull nginx:1.29.4-alpine-slim
	docker pull alpine:3.23
	shell: bash
	- name: Run integration tests
	run: make test-integration-ci
	env:
	EXTRA: cpu
	HF_TOKEN: ${{ secrets.HF_TOKEN }}
	NMP_FILES_HF_RETRY_ATTEMPTS: "7"
	NMP_FILES_HF_RETRY_INITIAL_DELAY_SECONDS: "1"
	NMP_FILES_HF_RETRY_MAX_DELAY_SECONDS: "30"
	PYTEST_WORKERS: "4"
	_TYPER_FORCE_DISABLE_TERMINAL: "1"
	# Per-worker crash dumps; pytest's own faulthandler would point them at stderr, which
	# xdist never forwards. See conftest.py.
	PYTEST_CRASH_DUMP_DIR: crash-dumps
	PYTEST_EXTRA: -p no:faulthandler
	- name: Upload test artifacts
	if: always()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: python-integration-test-results
	retention-days: 30
	path: \|
	report.xml
	coverage.xml
	coverage.json
	- name: Upload worker crash dumps
	# Separate artifact so coverage-comment keeps receiving exactly the files it expects.
	if: failure()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: python-integration-crash-dumps
	retention-days: 30
	if-no-files-found: ignore
	path: crash-dumps/

	python-auth-idp-static-test:
	name: Python auth-idp static tests
	needs: [changes]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.cpu-smoke == 'true' \|\|
	needs.changes.outputs.helm == 'true' \|\|
	needs.changes.outputs.auth-idp == 'true'
	)
	runs-on: ubuntu-latest
	permissions:
	contents: read
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Install uv
	uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
	with:
	version: ${{ env.UV_VERSION }}
	python-version: "3.12"
	enable-cache: true
	cache-dependency-glob: uv.lock
	- name: Run auth-idp static tests
	run: \|
	set -euo pipefail
	docker pull docker.io/envoyproxy/envoy:v1.37.0
	docker pull envoyproxy/envoy:v1.36.2
	helm dependency build k8s/helm
	helm dependency build contrib/auth/authentik/helm
	uv run --frozen pytest tests/auth_idp/static -v
	env:
	_TYPER_FORCE_DISABLE_TERMINAL: "1"
	- name: Upload test artifacts
	if: always()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: python-auth-idp-static-test-results
	retention-days: 30
	if-no-files-found: ignore
	path: \|
	report.xml
	coverage.xml
	coverage.json

	python-auth-idp-e2e-test:
	name: Python auth-idp ${{ matrix.runtime }} tests
	needs: [changes, policy-wasm, build-cpu-smoke-images, python-auth-idp-static-test]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.cpu-smoke == 'true' \|\|
	needs.changes.outputs.auth-idp == 'true'
	) &&
	needs.policy-wasm.result == 'success' &&
	needs.build-cpu-smoke-images.result == 'success' &&
	needs.build-cpu-smoke-images.outputs.publish_images == 'true' &&
	needs.python-auth-idp-static-test.result == 'success'
	runs-on: ubuntu-latest
	timeout-minutes: 90
	permissions:
	contents: read
	packages: read
	strategy:
	fail-fast: false
	matrix:
	include:
	- provider: authentik
	runtime: authentik-compose
	backend: compose
	command: compose
	- provider: authentik
	runtime: authentik-kubernetes
	backend: kubernetes
	command: k8s
	env:
	NMP_AUTHENTIK_K8S_RUNTIME: kind
	NMP_AUTHENTIK_K8S_NAMESPACE: nemo-authentik
	NMP_AUTHENTIK_K8S_IMAGE_PULL_SECRET: ghcr-pull
	NMP_AUTHENTIK_K8S_NGC_EXISTING_SECRET: ngc-api
	NMP_AUTHENTIK_K8S_CLUSTER_NAME: gha-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.runtime }}
	NMP_AUTHENTIK_K8S_REUSE_CLUSTER: "1"
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Free disk space
	uses: ./.github/actions/free-disk-space
	- name: Install uv
	uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
	with:
	version: ${{ env.UV_VERSION }}
	python-version: "3.12"
	enable-cache: true
	cache-dependency-glob: uv.lock
	- name: Setup Kind cluster
	if: matrix.backend == 'kubernetes'
	uses: ./.github/actions/setup-kind-cluster
	with:
	kind-cluster-name: ${{ env.NMP_AUTHENTIK_K8S_CLUSTER_NAME }}
	install-nemo-platform: "false"
	kube-namespace: ${{ env.NMP_AUTHENTIK_K8S_NAMESPACE }}
	image-registry: ${{ needs.build-cpu-smoke-images.outputs.image_registry }}
	image-tag: ${{ needs.build-cpu-smoke-images.outputs.image_tag }}
	kind-image-pull-token: ${{ github.token }}
	kind-image-pull-user: ${{ github.actor }}
	- name: Log in to GHCR
	if: matrix.backend == 'compose' && needs.build-cpu-smoke-images.outputs.publish_images == 'true'
	uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
	with:
	registry: ghcr.io
	username: ${{ github.actor }}
	password: ${{ github.token }}
	- name: Install Helm
	if: matrix.backend == 'kubernetes'
	uses: azure/setup-helm@dda3372f752e03dde6b3237bc9431cdc2f7a02a2 # v5.0.0
	- name: Lint Authentik umbrella Helm chart
	if: matrix.provider == 'authentik' && matrix.backend == 'kubernetes'
	shell: bash
	run: \|
	helm repo add authentik https://charts.goauthentik.io --force-update
	helm dependency build k8s/helm
	helm dependency build contrib/auth/authentik/helm
	helm lint --strict contrib/auth/authentik/helm
	- name: Verify official Authentik Helm chart is reachable
	if: matrix.provider == 'authentik' && matrix.backend == 'kubernetes'
	shell: bash
	run: \|
	helm show chart authentik --repo https://charts.goauthentik.io --version 2026.5.4
	- name: Download policy WASM
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: policy-wasm
	path: services/core/auth/src/nmp/core/auth/assets
	- name: Run auth-idp ${{ matrix.runtime }} tests
	run: >
	contrib/auth/${{ matrix.provider }}/run.sh
	${{ matrix.command }}
	--image "${IMAGE_REGISTRY}/nmp-api:${BAKE_TAG}"
	env:
	_TYPER_FORCE_DISABLE_TERMINAL: "1"
	E2E_SERVICES_LOG_DIR: ${{ runner.temp }}/e2e-services-logs
	IMAGE_REGISTRY: ${{ needs.build-cpu-smoke-images.outputs.image_registry }}
	BAKE_TAG: ${{ needs.build-cpu-smoke-images.outputs.image_tag }}
	NMP_AUTHENTIK_K8S_JUNIT_XML: report-auth-idp-${{ matrix.runtime }}.xml
	NMP_AUTHENTIK_K8S_IMAGE_PULL_SECRET: ${{ env.NMP_AUTHENTIK_K8S_IMAGE_PULL_SECRET }}
	NMP_AUTHENTIK_K8S_SKIP_IMAGE_LOAD: ${{ needs.build-cpu-smoke-images.outputs.publish_images == 'true' && '1' \|\| '0' }}
	- name: Collect Kubernetes logs
	if: always() && matrix.backend == 'kubernetes'
	shell: bash
	run: e2e/k8s/scripts/collect_k8s_logs.sh
	- name: Upload test artifacts
	if: always()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: python-auth-idp-${{ matrix.runtime }}-test-results
	retention-days: 30
	if-no-files-found: ignore
	path: \|
	k8s-logs/
	report-auth-idp-${{ matrix.runtime }}.xml
	${{ runner.temp }}/e2e-services-logs/
	- name: Delete kind cluster
	if: always() && matrix.backend == 'kubernetes'
	shell: bash
	run: \|
	docker rm -f "cloud-provider-kind-${NMP_AUTHENTIK_K8S_CLUSTER_NAME}" \|\| true
	kind delete cluster --name "${NMP_AUTHENTIK_K8S_CLUSTER_NAME}" \|\| true

	# Build wheels for all packages × python versions. Downstream jobs
	# (wheel-test, python-e2e-test) download these artifacts instead
	# of rebuilding.
	wheel-build:
	name: ${{ matrix.package }} wheel build (py${{ matrix.python-version }})
	needs: [policy-wasm]
	runs-on: ubuntu-latest
	timeout-minutes: 25
	permissions:
	contents: read
	strategy:
	fail-fast: false
	matrix:
	package: [nemo-platform, nemo-platform-plugin]
	python-version: ["3.12", "3.13"]
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Download policy WASM
	if: matrix.package == 'nemo-platform'
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: policy-wasm
	path: services/core/auth/src/nmp/core/auth/assets
	- name: Build ${{ matrix.package }} wheel
	id: build
	uses: ./.github/actions/build-nemo-platform-wheel
	with:
	package: ${{ matrix.package }}
	out-dir: ${{ github.workspace }}/dist
	cadence: nightly
	# Sentinel epoch — the wheel version is meaningless for a CI test
	# build, but stamp_sdk_version.py requires \d{14}.
	nightly-timestamp: "19700101000000"
	python-version: ${{ matrix.python-version }}
	- name: Upload wheel
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: ${{ matrix.package }}-wheel-py${{ matrix.python-version }}
	path: ${{ steps.build.outputs.wheel-path }}
	retention-days: 7
	if-no-files-found: error

	# Test each wheel: install from the artifact and run basic CLI /
	# import checks.
	wheel-test:
	name: ${{ matrix.package }} wheel build + test (py${{ matrix.python-version }}, ${{ matrix.install-method }})
	needs: [wheel-build]
	runs-on: ubuntu-latest
	timeout-minutes: 15
	permissions:
	contents: read
	strategy:
	fail-fast: false
	matrix:
	package: [nemo-platform, nemo-platform-plugin]
	python-version: ["3.12", "3.13"]
	# `uv tool install` is the documented front door for the CLI; `venv`
	# covers installs into an environment the user owns. The plugin is a
	# library with no CLI entry point, so it only gets the venv leg.
	install-method: [venv, tool]
	exclude:
	- package: nemo-platform-plugin
	install-method: tool
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Install uv
	uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
	with:
	version: ${{ env.UV_VERSION }}
	python-version: ${{ matrix.python-version }}
	enable-cache: true
	cache-dependency-glob: uv.lock
	- name: Download wheel
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: ${{ matrix.package }}-wheel-py${{ matrix.python-version }}
	path: ${{ runner.temp }}/wheelcheck

	- name: Install + test nemo-platform CLI
	if: matrix.package == 'nemo-platform'
	shell: bash
	env:
	PYTHON_VERSION: ${{ matrix.python-version }}
	INSTALL_METHOD: ${{ matrix.install-method }}
	NMP_DATA_DIR: ${{ runner.temp }}/nemo-data
	NMP_AUTH_ENABLED: "false"
	_TYPER_FORCE_DISABLE_TERMINAL: "1"
	UV_TOOL_BIN_DIR: ${{ runner.temp }}/wheelcheck/tool-bin
	run: \|
	set -euo pipefail
	shopt -s nullglob
	wheels=("${RUNNER_TEMP}"/wheelcheck/*.whl)
	if [[ "${#wheels[@]}" -ne 1 ]]; then
	echo "::error::Expected exactly one wheel, found ${#wheels[@]}"
	exit 1
	fi
	WHEEL="${wheels[0]}"
	# Supply-chain: partial-pin the wheel's direct external deps to a committed constraints
	# file. Regenerate with script/compile-wheel-constraints.sh.
	CONSTRAINTS="${GITHUB_WORKSPACE}/.github/wheel-constraints/nemo-platform-services.txt"
	# --no-config: this step runs inside the checkout, so uv would otherwise apply the
	# repo's [tool.uv] override-dependencies and resolve a dependency set no user can get.
	if [[ "${INSTALL_METHOD}" == "tool" ]]; then
	uv tool install --no-config --python "${PYTHON_VERSION}" \
	--constraints "${CONSTRAINTS}" \
	"${WHEEL}[services]"
	BIN_DIR="${UV_TOOL_BIN_DIR}"
	else
	VENV="${RUNNER_TEMP}/wheelcheck/.venv"
	uv venv "${VENV}" --python "${PYTHON_VERSION}"
	uv pip install --no-config --python "${VENV}/bin/python" \
	--constraint "${CONSTRAINTS}" \
	"${WHEEL}[services]"
	BIN_DIR="${VENV}/bin"
	fi
	cd "${RUNNER_TEMP}/wheelcheck"
	unset PYTHONPATH VIRTUAL_ENV
	export PATH="${BIN_DIR}:${PATH}"
	bash "${GITHUB_WORKSPACE}/script/test-nemo-cli.sh"

	- name: Install + test nemo-platform-plugin
	if: matrix.package == 'nemo-platform-plugin'
	shell: bash
	env:
	PYTHON_VERSION: ${{ matrix.python-version }}
	run: \|
	set -euo pipefail
	shopt -s nullglob
	wheels=("${RUNNER_TEMP}"/wheelcheck/*.whl)
	if [[ "${#wheels[@]}" -ne 1 ]]; then
	echo "::error::Expected exactly one wheel, found ${#wheels[@]}"
	exit 1
	fi
	WHEEL="${wheels[0]}"
	# Supply-chain: partial-pin direct external deps via a committed constraints file
	# (see the nemo-platform CLI step above).
	cd "${RUNNER_TEMP}/wheelcheck"
	uv venv .venv --python "${PYTHON_VERSION}"
	uv pip install --no-config --python .venv/bin/python \
	--constraint "${GITHUB_WORKSPACE}/.github/wheel-constraints/nemo-platform-plugin.txt" \
	"${WHEEL}"
	unset PYTHONPATH VIRTUAL_ENV
	# Import the package + a couple of representative submodules
	# (cli, commands). These are the surfaces a plugin author would
	# touch first; if any of them fail to import, the wheel is
	# broken in a way that surfaces immediately on day one.
	.venv/bin/python -c "
	import nemo_platform_plugin
	import nemo_platform_plugin.cli
	import nemo_platform_plugin.commands
	print('nemo_platform_plugin', getattr(nemo_platform_plugin, '__version__', '<no __version__>'))
	"

	# E2E tests: start services from the workspace venv and run the e2e
	# suite against the real process.
	python-e2e-test:
	name: Python e2e tests
	needs: [policy-wasm]
	runs-on: ubuntu-latest
	timeout-minutes: 15
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Download policy WASM
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: policy-wasm
	path: services/core/auth/src/nmp/core/auth/assets
	- name: Install uv
	uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
	with:
	version: ${{ env.UV_VERSION }}
	python-version: "3.13"
	enable-cache: true
	cache-dependency-glob: uv.lock
	- name: Run e2e tests
	run: make test-e2e
	env:
	_TYPER_FORCE_DISABLE_TERMINAL: "1"
	E2E_SERVICES_LOG_DIR: ${{ runner.temp }}/e2e-services-logs
	NGC_API_KEY: ${{ secrets.NGC_REGISTRY_READ_TOKEN }}
	HF_TOKEN: ${{ secrets.HF_TOKEN }}
	NMP_FILES_HF_RETRY_ATTEMPTS: "7"
	NMP_FILES_HF_RETRY_INITIAL_DELAY_SECONDS: "1"
	NMP_FILES_HF_RETRY_MAX_DELAY_SECONDS: "30"
	- name: Dump server logs
	if: always()
	run: \|
	echo "::group::Server logs"
	for f in "${{ runner.temp }}/e2e-services-logs"/*.log; do
	[ -f "$f" ] && echo "--- $(basename "$f") ---" && cat "$f" \|\| echo "No server logs found"
	done
	echo "::endgroup::"
	- name: Upload test artifacts
	if: always()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: python-e2e-test-results
	retention-days: 30
	path: \|
	report.xml
	${{ runner.temp }}/e2e-services-logs/

	# E2E tests that run on the subprocess harness but need a prebuilt platform
	# image available in the local Docker daemon (e.g. docker-mode agent
	# deployments, which deploy the nmp-api image as a sibling container). Marked
	# `needs_nmp_api_image`; the plain python-e2e-test job skips them because
	# NMP_E2E_IMAGE_REGISTRY / NMP_E2E_IMAGE_TAG are unset there.
	python-e2e-image-test:
	name: Python e2e tests (prebuilt image)
	needs: [policy-wasm, build-cpu-smoke-images]
	if: >
	!cancelled() &&
	needs.build-cpu-smoke-images.result == 'success' &&
	needs.build-cpu-smoke-images.outputs.publish_images == 'true'
	runs-on: ubuntu-latest
	timeout-minutes: 20
	permissions:
	contents: read
	packages: read
	env:
	NMP_E2E_IMAGE_REGISTRY: ${{ needs.build-cpu-smoke-images.outputs.image_registry }}
	NMP_E2E_IMAGE_TAG: ${{ needs.build-cpu-smoke-images.outputs.image_tag }}
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Download policy WASM
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: policy-wasm
	path: services/core/auth/src/nmp/core/auth/assets
	- name: Install uv
	uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
	with:
	version: ${{ env.UV_VERSION }}
	python-version: "3.13"
	enable-cache: true
	cache-dependency-glob: uv.lock
	- name: Log in to GHCR
	shell: bash
	env:
	GHCR_TOKEN: ${{ github.token }}
	run: echo "$GHCR_TOKEN" \| docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin
	- name: Pull nmp-api image (timed)
	shell: bash
	run: \|
	image="${NMP_E2E_IMAGE_REGISTRY}/nmp-api:${NMP_E2E_IMAGE_TAG}"
	echo "Pulling ${image}"
	start=$(date +%s)
	docker pull "$image"
	echo "nmp-api pull took $(( $(date +%s) - start ))s"
	- name: Run e2e tests (needs_nmp_api_image)
	run: make test-e2e PYTEST_EXTRA="-m needs_nmp_api_image"
	env:
	_TYPER_FORCE_DISABLE_TERMINAL: "1"
	E2E_SERVICES_LOG_DIR: ${{ runner.temp }}/e2e-services-logs
	- name: Dump server logs
	if: always()
	run: \|
	echo "::group::Server logs"
	for f in "${{ runner.temp }}/e2e-services-logs"/*.log; do
	[ -f "$f" ] && echo "--- $(basename "$f") ---" && cat "$f" \|\| echo "No server logs found"
	done
	echo "::endgroup::"
	- name: Upload test artifacts
	if: always()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: python-e2e-image-test-results
	retention-days: 30
	path: \|
	report.xml
	${{ runner.temp }}/e2e-services-logs/

	web-typecheck:
	name: Web typecheck
	needs: [changes]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.web-studio == 'true' \|\|
	needs.changes.outputs.openapi == 'true'
	)
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: web
	shell: bash
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
	with:
	node-version-file: .nvmrc
	- name: Install pnpm via corepack
	run: npm i -g corepack@0.31.0 && corepack enable pnpm
	- name: Install dependencies
	run: pnpm install --frozen-lockfile
	- name: Typecheck web packages
	run: \|
	if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ needs.changes.outputs.openapi }}" != "true" ]; then
	git fetch origin "${{ github.base_ref }}" --depth=1
	pnpm --filter="...[origin/${{ github.base_ref }}]" run --parallel --if-present typecheck
	else
	pnpm run --recursive --parallel --if-present typecheck
	fi

	web-test:
	name: Web tests
	needs: [changes]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.web-studio == 'true'
	)
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: web
	shell: bash
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
	with:
	node-version-file: .nvmrc
	- name: Install pnpm via corepack
	run: npm i -g corepack@0.31.0 && corepack enable pnpm
	- name: Install dependencies
	run: pnpm install --frozen-lockfile
	- name: Test changed packages
	run: \|
	if [ "${{ github.event_name }}" = "pull_request" ]; then
	git fetch origin "${{ github.base_ref }}" --depth=1
	pnpm --filter="...[origin/${{ github.base_ref }}]" run --parallel --if-present test:ci
	else
	pnpm run --recursive --parallel --if-present test:ci
	fi

	web-format:
	name: Web format check
	needs: [changes]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.web-studio == 'true'
	)
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: web
	shell: bash
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
	with:
	node-version-file: .nvmrc
	- name: Install pnpm via corepack
	run: npm i -g corepack@0.31.0 && corepack enable pnpm
	- name: Install dependencies
	run: pnpm install --frozen-lockfile
	- name: Check formatting
	run: pnpm format

	web-lint:
	name: Web lint
	needs: [changes]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.web-studio == 'true'
	)
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: web
	shell: bash
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
	with:
	node-version-file: .nvmrc
	- name: Install pnpm via corepack
	run: npm i -g corepack@0.31.0 && corepack enable pnpm
	- name: Install dependencies
	run: pnpm install --frozen-lockfile
	- name: Lint
	run: pnpm lint

	web-sdk-gen:
	name: Web SDK generation check
	needs: [changes]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.openapi == 'true' \|\|
	needs.changes.outputs.web-studio == 'true'
	)
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: web
	shell: bash
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
	with:
	node-version-file: .nvmrc
	- name: Install pnpm via corepack
	run: npm i -g corepack@0.31.0 && corepack enable pnpm
	- name: Install dependencies
	run: pnpm install --frozen-lockfile
	- name: Generate SDK from OpenAPI specs
	run: pnpm --filter @nemo/sdk gen:all-force
	- name: Typecheck SDK generated output
	run: pnpm --filter @nemo/sdk typecheck
	- name: Typecheck web packages against regenerated SDK
	run: pnpm run --recursive --parallel --if-present typecheck

	web-plugin-types:
	name: Web plugin surface types check
	needs: [changes]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.web-studio == 'true'
	)
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: web
	shell: bash
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
	with:
	node-version-file: .nvmrc
	- name: Install pnpm via corepack
	run: npm i -g corepack@0.31.0 && corepack enable pnpm
	- name: Install dependencies
	run: pnpm install --frozen-lockfile
	- name: Regenerate the plugin surface types
	run: pnpm --filter @nemo/common types:plugin
	# Out-of-tree plugins vendor this file, so a stale copy types them against
	# a surface that no longer exists — and still compiles, which is the whole
	# problem. Fail the change that moved the surface, not the plugin later.
	- name: Check the committed artifact is up to date
	run: git diff --exit-code -- packages/common/plugin-types

	web-studio-deps:
	name: Web studio deps check
	needs: [changes]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.web-studio == 'true'
	)
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: web
	shell: bash
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
	with:
	node-version-file: .nvmrc
	- name: Install pnpm via corepack
	run: npm i -g corepack@0.31.0 && corepack enable pnpm
	- name: Install dependencies
	run: pnpm install --frozen-lockfile
	- name: Check studio deps
	run: pnpm deps:studio

	web-studio-e2e:
	name: Studio UI E2E tests
	if: github.event_name == 'workflow_dispatch'
	runs-on: ubuntu-latest
	defaults:
	run:
	working-directory: web
	shell: bash
	steps:
	- uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
	with:
	node-version-file: .nvmrc
	- name: Install pnpm via corepack
	run: npm i -g corepack@0.31.0 && corepack enable pnpm
	- name: Install dependencies
	run: pnpm install --frozen-lockfile
	- name: Install Playwright browsers
	run: pnpm --filter nemo-studio-ui exec playwright install --with-deps
	- name: Run E2E tests
	run: CI=1 pnpm --filter nemo-studio-ui test:e2e
	- name: Upload E2E artifacts
	if: always()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	name: studio-e2e-results
	retention-days: 7
	path: web/packages/studio/playwright-report/

	deployments-openshell-tests:
	# The openshell extra is a platform-restricted manylinux_2_39 wheel that the
	# shared unit-test job does not install, so the OpenShell backend tests skip
	# there. Install the extra in this isolated, path-filtered job and exercise the
	# backend for real. Kept separate so a wheel-install failure fails only this
	# small job, not the whole unit suite (mirrors the nemo-guardrails extra job).
	name: Deployments OpenShell backend tests
	needs: [changes]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.deployments-openshell == 'true'
	)
	runs-on: ubuntu-latest
	timeout-minutes: 15
	permissions:
	contents: read
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Install uv
	# No python-version pin: uv resolves the interpreter from the workspace
	# requires-python (>=3.12), so this job can never fall back to an
	# unsupported version.
	uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
	with:
	version: ${{ env.UV_VERSION }}
	enable-cache: true
	cache-dependency-glob: uv.lock
	- name: Run OpenShell backend unit tests
	run: make test-deployments-openshell
	env:
	_TYPER_FORCE_DISABLE_TERMINAL: "1"

	guardrails-benchmark:
	# Parallel matrix jobs (one NMP per variant) so the two sweeps don't
	# share mocks or contend on :8080. `guardrails-benchmark-analyze` merges
	# the artifacts and prints the comparison.
	name: nemo-guardrails plugin benchmark (${{ matrix.variant }})
	needs: [changes]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.guardrails-benchmark == 'true'
	)
	runs-on: ubuntu-latest
	timeout-minutes: 30
	strategy:
	# Keep the partial artifact if one variant fails.
	fail-fast: false
	matrix:
	variant: [with-guardrails, without-guardrails]
	steps:
	- name: Checkout nemo-platform
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	path: nemo-platform
	persist-credentials: false
	- name: Checkout NeMo-Guardrails
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	repository: NVIDIA/NeMo-Guardrails
	path: NeMo-Guardrails
	persist-credentials: false
	- name: Install uv
	uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
	with:
	working-directory: nemo-platform
	version: ${{ env.UV_VERSION }}
	python-version: "3.12"
	enable-cache: true
	cache-dependency-glob: uv.lock
	- name: Bootstrap Python environment
	working-directory: nemo-platform
	run: make bootstrap-python
	env:
	PYTORCH_DEPS: cpu
	- name: Run benchmark sweep
	working-directory: nemo-platform
	# Pin both variants to the same `--run-id` so when the analyze job
	# downloads both artifacts into one `runs/` parent, they merge into
	# a single run directory the analyzer can read normally.
	run: \|
	make benchmark-guardrails BENCHMARK_ARGS="\
	--variant ${{ matrix.variant }} \
	--run-id ci-${{ github.run_id }}-${{ github.run_attempt }}"
	env:
	NEMO_GUARDRAILS_REPO_ROOT: ${{ github.workspace }}/NeMo-Guardrails
	_TYPER_FORCE_DISABLE_TERMINAL: "1"
	- name: Upload benchmark artifacts
	if: always()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	# Ensure we use a unique artifact name per benchmark variant.
	name: guardrails-benchmark-results-${{ matrix.variant }}
	retention-days: 30
	path: \|
	nemo-platform/plugins/nemo-guardrails/benchmarks/artifacts/runs/

	guardrails-benchmark-analyze:
	# Merge both variant artifacts and print the comparison table.
	name: nemo-guardrails plugin benchmark analysis
	needs: [changes, guardrails-benchmark]
	if: >
	!cancelled() && (
	github.event_name == 'workflow_dispatch' \|\|
	needs.changes.outputs.guardrails-benchmark == 'true'
	)
	runs-on: ubuntu-latest
	timeout-minutes: 10
	steps:
	- name: Checkout nemo-platform
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	path: nemo-platform
	persist-credentials: false
	- name: Download with-guardrails artifact
	# If a variant failed entirely it may have uploaded no artifact;
	# the analyzer handles the single-variant case so don't fail here.
	continue-on-error: true
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: guardrails-benchmark-results-with-guardrails
	path: nemo-platform/plugins/nemo-guardrails/benchmarks/artifacts/runs/
	- name: Download without-guardrails artifact
	continue-on-error: true
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: guardrails-benchmark-results-without-guardrails
	path: nemo-platform/plugins/nemo-guardrails/benchmarks/artifacts/runs/
	- name: Print benchmark comparison
	working-directory: nemo-platform
	# `analyze.py` doesn't rely on NMP or AIPerf, so we skip the uv bootstrap
	# step and run it with the runner's `python3` CLI directly.
	run: \|
	RUN_DIR=$(find plugins/nemo-guardrails/benchmarks/artifacts/runs -mindepth 1 -maxdepth 1 -type d -printf '%T@ %p\n' \| sort -nr \| head -1 \| cut -d' ' -f2-)
	echo "Analyzing run directory: $RUN_DIR"
	python3 plugins/nemo-guardrails/src/nemo_guardrails_plugin/benchmarks/analyze.py "$RUN_DIR" --strict
	- name: Upload merged benchmark artifacts
	if: always()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
	with:
	# Single artifact so baseline collection is one download per run.
	name: guardrails-benchmark-results-merged
	retention-days: 30
	path: \|
	nemo-platform/plugins/nemo-guardrails/benchmarks/artifacts/runs/

	coverage-comment:
	name: Post coverage comment
	needs: [python-unit-test, python-integration-test]
	if: >
	always() &&
	github.event_name == 'pull_request' &&
	(
	needs.python-unit-test.result == 'success' \|\|
	needs.python-unit-test.result == 'failure' \|\|
	needs.python-integration-test.result == 'success' \|\|
	needs.python-integration-test.result == 'failure'
	)
	runs-on: ubuntu-latest
	permissions:
	pull-requests: write
	contents: read
	steps:
	- name: Download unit test coverage
	continue-on-error: true
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: python-unit-test-results
	path: unit-coverage
	- name: Download integration test coverage
	continue-on-error: true
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
	with:
	name: python-integration-test-results
	path: integration-coverage
	- name: Prepare coverage table
	run: \|
	extract_coverage() {
	local xml_file="$1"
	local label="$2"
	if [[ ! -f "${xml_file}" ]]; then
	echo "\| ${label} \| N/A \| N/A \| N/A \|"
	return
	fi
	LINE_RATE=$(grep -oP 'line-rate="\K[^"]+' "${xml_file}" \| head -1)
	LINE_COVERED=$(grep -oP 'lines-covered="\K[^"]+' "${xml_file}" \| head -1)
	LINE_VALID=$(grep -oP 'lines-valid="\K[^"]+' "${xml_file}" \| head -1)
	BRANCH_RATE=$(grep -oP 'branch-rate="\K[^"]+' "${xml_file}" \| head -1)
	LINE_PCT=$(awk "BEGIN {printf \"%.1f\", ${LINE_RATE} * 100}")
	BRANCH_PCT=$(awk "BEGIN {printf \"%.1f\", ${BRANCH_RATE} * 100}")
	echo "\| ${label} \| ${LINE_COVERED}/${LINE_VALID} \| ${LINE_PCT}% \| ${BRANCH_PCT}% \|"
	}

	{
	echo "\| Suite \| Lines Covered \| Line Rate \| Branch Rate \|"
	echo "\| --- \| --- \| --- \| --- \|"
	extract_coverage "unit-coverage/coverage.xml" "Unit Tests"
	extract_coverage "integration-coverage/coverage.xml" "Integration Tests"
	} > summary.md
	cat summary.md
	shell: bash
	- name: Post comment
	uses: marocchino/sticky-pull-request-comment@70d2764d1a7d5d9560b100cbea0077fc8f633987
	with:
	header: coverage-report
	path: summary.md

	opa-policy-test:
	name: Test OPA Policy
	runs-on: ubuntu-latest
	steps:
	- name: Checkout code
	uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
	with:
	persist-credentials: false
	- name: Install Deps
	uses: flox/install-flox-action@c94e7e1ab56ae14fe98bae4fd84384fd135f0c2a # v2.4.0
	with:
	disable-metrics: "true"
	- name: Policy Test
	uses: flox/activate-action@410568008895a0f2e09a34bbd9523f8ef1f2d292 # v1.1.0
	with:
	dir: ./tools/open-policy-agent
	command: \|
	opa test services/core/auth/src/nmp/core/auth/app/policies services/core/auth/src/nmp/core/auth/app/policy_tests services/core/auth/src/nmp/core/auth/assets/static-authz.yaml -v

	# Required-check pin: branch protection should require this final aggregate.
	# Skipped jobs count as pass so conditional jobs can remain optional.
	ci-status:
	name: CI status
	needs:
	- changes
	- actionlint
	- uv-lock
	- uv-compatibility
	- docker-bake-graph
	- build-cpu-smoke-images
	- kind-cpu-smoke
	- kind-gym-e2e
	- kind-network-policy-smoke
	- helm-lint
	- helm-chart-verifier
	- lint
	- ngc-metadata-test
	- policy-wasm
	- python-unit-test-tools
	- python-unit-test
	- fabric-wheel-smoke
	- evaluator-sdk-closure-smoke
	# Enable if you want this required
	# - python-integration-test
	- wheel-build
	- wheel-test
	- python-e2e-test
	- web-typecheck
	- web-test
	- web-format
	- web-lint
	- web-sdk-gen
	- web-plugin-types
	- web-studio-deps
	- web-studio-e2e
	- opa-policy-test
	- guardrails-benchmark
	- guardrails-benchmark-analyze
	if: always()
	runs-on: ubuntu-latest
	permissions:
	contents: read
	steps:
	- name: Check all dependent jobs
	shell: bash
	env:
	NEEDS_JSON: ${{ toJson(needs) }}
	run: \|
	set -euo pipefail

	{
	echo "## CI status"
	echo
	echo "\| Job \| Result \|"
	echo "\| --- \| --- \|"
	printf '%s\n' "${NEEDS_JSON}" \
	\| jq -r 'to_entries[] \| "\| \(.key) \| \(.value.result) \|"'
	} >> "${GITHUB_STEP_SUMMARY}"

	printf '%s\n' "${NEEDS_JSON}" \
	\| jq -e 'to_entries \| map(.value.result) \| all(. as $result \| ["success", "skipped"] \| any($result == .))'

Navigation Menu

Uh oh!

refactor(evaluator)!: make runner and agent-eval metrics built-in (#1… #9298

Workflow file

refactor(evaluator)!: make runner and agent-eval metrics built-in (#1… #9298

Uh oh!

Workflow file for this run