Skip to content
build

feat(tests): add agentic scope test coverage #431

Sign in to view logs

feat(tests): add agentic scope test coverage

feat(tests): add agentic scope test coverage #431

Workflow file for this run

.github/workflows/ci.yml at 3f861d1
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
name: build
on:
push:
branches:
- main
- "feature/**"
- "feat/**"
- "fix/**"
- "chore/**"
- "docs/**"
- "refactor/**"
- "ci/**"
- "hotfix/**"
pull_request:
branches:
- main
# schedule:
# # Nightly load test at 02:00 UTC — runs against a CI-local gateway instance.
# - cron: "0 2 * * *"
# Default least-privilege permissions for every job in this workflow.
# zizmor's excessive-permissions audit flags jobs with no permissions:
# block because they inherit the (potentially broad) repository default.
# Individual jobs may still narrow further if they need write access.
permissions:
contents: read
jobs:
squash-merge-guard:
name: "Squash Merge Guard"
runs-on: ubuntu-latest
if: github.ref == 'refs/heads/main' && github.event_name == 'push'
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 2
persist-credentials: false
- name: Detect non-squash merge commits on main
run: |
parents=$(git cat-file -p HEAD | grep -c "^parent ")
if [ "$parents" -gt 1 ]; then
echo "::error::Non-squash merge commit detected on main ($(git rev-parse HEAD))."
echo "::error::GIT_WORKFLOW_STANDARDS §4.5 requires squash merge for all PRs."
echo "::error::Fix: Settings → General → Pull Requests → disable 'Allow merge commits'."
exit 1
fi
echo "OK: HEAD is a single-parent commit (squash or direct push)."
license-check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Check Apache 2.0 License Headers
run: |
# A simple check to ensure Apache 2.0 or Copyright 2026 Cybernetic Governance Engine is present
echo "Validating Apache 2.0 licensing headers across all source files..."
missing_headers=0
for file in $(find src/ -name '*.py' -o -name '*.js' -o -name '*.ts' -o -name '*.tsx'); do
if ! grep -q "Apache License" "$file" && ! grep -q "Copyright 2026" "$file"; then
echo "Missing Apache 2.0 license header in: $file"
missing_headers=1
fi
done
if [ "$missing_headers" -eq 1 ]; then
echo "License validation failed."
exit 1
fi
echo "All source files have the required Apache 2.0 licensing headers."
pytest-logic:
name: "Pytest Logic Tests (${{ matrix.region }})"
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
region: [US_FED, EU_ECB, APAC_MAS]
include:
# Branch coverage is region-agnostic — run it only on US_FED to save
# ~15-20% wall time on the EU_ECB and APAC_MAS legs.
- region: US_FED
cov_branch_flag: "--cov-branch"
- region: EU_ECB
cov_branch_flag: ""
- region: APAC_MAS
cov_branch_flag: ""
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
with:
enable-cache: true
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version-file: "pyproject.toml"
- name: Install Dependencies
run: uv sync --all-groups --all-extras
- name: Run Pytest (unit/local)
env:
CAGE_DEPLOYMENT_REGION: ${{ matrix.region }}
CAGE_ENV: "test"
OPENAI_API_KEY: "sk-dummy"
MODEL_REASONING: "casperhansen/deepseek-r1-distill-qwen-14b-awq"
MODEL_FAST: "Qwen/Qwen2.5-7B-Instruct"
VLLM_API_KEY: "dummy"
VLLM_GATEWAY_URL: "http://localhost:8081/v1"
REDIS_URL: "redis://localhost:6379/0"
OPA_URL: "http://localhost:8181/v1/data/governance/policy"
LANGFUSE_HOST: "http://localhost:3000"
LANGFUSE_PUBLIC_KEY: "pk-dummy"
LANGFUSE_SECRET_KEY: "sk-dummy"
run: uv run pytest tests/ -m "local or unit" -n auto --dist=loadfile -v ${{ matrix.cov_branch_flag }} --cov=src --cov-fail-under=70
- name: Run Bandit SAST (medium+ severity)
run: uv run bandit -r src/ -c pyproject.toml -ll
lint:
name: "Lint"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
with:
enable-cache: true
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version-file: "pyproject.toml"
- name: Install dependencies
run: uv sync --all-groups --all-extras
- name: Ruff lint
run: uv run ruff check .
- name: Ruff format check
run: uv run ruff format --check .
- name: Mypy type check
run: uv run mypy src/
eu-ecb-bias-eval:
name: EU_ECB LLM Bias Evaluation
runs-on: ubuntu-latest
if: vars.CAGE_DEPLOYMENT_REGION == 'EU_ECB'
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
with:
version: "latest"
- name: Install dependencies
run: uv sync --all-groups --all-extras
- name: Run EU_ECB bias metrics tests
env:
CAGE_DEPLOYMENT_REGION: EU_ECB
VLLM_FAST_API_BASE: ${{ vars.VLLM_FAST_API_BASE || 'http://localhost:8001/v1' }}
run: |
pytest compliance/postures/eu_ecb/llm_eval/ -v --timeout=300
stpa-freshness-check:
name: "STPA Artifact Freshness Check"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.11"
- name: Check generated STPA artifacts are current
run: |
python scripts/check_stpa_freshness.py --verbose
nemo-freshness-check:
name: "NeMo Rail Actions Freshness Check"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
fetch-depth: 0
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.11"
- name: Check nemo-rails-configmap actions.py snapshot is current
run: |
python3 - <<'EOF'
import sys
CONFIGMAP = "deployment/k8s/nemo-rails-configmap.yaml"
CANONICAL = "config/rails/actions.py"
# Extract the actions.py block from the ConfigMap YAML.
# The block starts at the line containing "actions.py: |" (or "actions.py: |-")
# and ends when the indentation drops back to the data: level.
try:
with open(CONFIGMAP) as f:
lines = f.readlines()
except FileNotFoundError:
print(f"ERROR: {CONFIGMAP} not found — add the configmap snapshot or update this check.")
sys.exit(1)
snapshot_lines = []
in_block = False
block_indent = None
for line in lines:
stripped = line.lstrip()
indent = len(line) - len(stripped)
if not in_block:
if stripped.startswith("actions.py:"):
in_block = True
# block_indent is the indentation of the content lines (one level deeper)
block_indent = indent + 2
continue
# Inside the block: collect lines that are indented at block_indent or deeper
if stripped == "" or indent >= block_indent:
snapshot_lines.append(line[block_indent:] if stripped else "\n")
else:
break # dedented back — end of block
if not snapshot_lines:
print(f"ERROR: Could not extract actions.py content from {CONFIGMAP}.")
print("Ensure the ConfigMap has a 'actions.py:' key under 'data:'.")
sys.exit(1)
snapshot = "".join(snapshot_lines).rstrip("\n") + "\n"
try:
with open(CANONICAL) as f:
canonical = f.read()
except FileNotFoundError:
print(f"ERROR: {CANONICAL} not found.")
sys.exit(1)
if snapshot == canonical:
print(f"OK: {CONFIGMAP} actions.py snapshot matches {CANONICAL}.")
else:
import difflib
diff = list(difflib.unified_diff(
snapshot.splitlines(keepends=True),
canonical.splitlines(keepends=True),
fromfile=f"{CONFIGMAP}:actions.py",
tofile=CANONICAL,
))
print("ERROR: ConfigMap actions.py snapshot is stale relative to config/rails/actions.py.")
print("Regenerate the snapshot: copy config/rails/actions.py into the ConfigMap data block.")
print("".join(diff[:80]))
sys.exit(1)
EOF
no-direct-bind-proof:
name: "NoDirectBind State-Space Proof"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
with:
enable-cache: true
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version-file: "pyproject.toml"
- name: Install Dependencies
run: uv sync --all-groups --all-extras
- name: Run exhaustive NoDirectBind enumeration
run: |
uv run python proof/model.py
- name: Run pinned NoDirectBind regression tests
run: |
uv run pytest tests/test_no_direct_bind_proof.py -m local -v -o addopts=""
langfuse-posture-check:
name: "Langfuse Posture Dry-Run Check"
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.11"
- name: Langfuse posture dry-run check
run: |
export GOOGLE_CLOUD_PROJECT="mock-dev-project"
# DEP-24: GOOGLE_CLOUD_LOCATION is derived from CAGE_DEPLOYMENT_REGION.
# US_FED → us-central1 | EU_ECB → europe-west1 | APAC_MAS → asia-southeast1
# Defaults to us-central1 for CI dry-run (no live cluster).
_region="${CAGE_DEPLOYMENT_REGION:-US_FED}"
case "$_region" in
EU_ECB) export GOOGLE_CLOUD_LOCATION="europe-west1" ;;
APAC_MAS) export GOOGLE_CLOUD_LOCATION="asia-southeast1" ;;
*) export GOOGLE_CLOUD_LOCATION="us-central1" ;;
esac
export LANGFUSE_HOST="http://localhost:3000"
export LANGFUSE_PUBLIC_KEY="pk-lf-mock"
export LANGFUSE_SECRET_KEY="sk-lf-mock"
export LANGFUSE_COMPLIANCE_HOST="http://localhost:3001"
export LANGFUSE_COMPLIANCE_PUBLIC_KEY="pk-lf-comp-mock"
export LANGFUSE_COMPLIANCE_SECRET_KEY="sk-lf-comp-mock"
python3 scripts/verify_langfuse_posture.py --dry-run --posture development
# ── AI 600-1 Compliance Gates ──────────────────────────────────────────────
lula-ai600-validation:
name: "Lula AI 600-1 Validation"
runs-on: ubuntu-latest
needs: [pytest-logic]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.11"
- name: Install PyYAML
run: pip install pyyaml
- name: Validate AI 600-1 Lula manifest structure (stub syntax check)
# Stub manifests use OSCAL component-definition structure (not standalone domain/provider).
# The domain/provider are embedded inside back-matter.resources[].description.
# Full live lula validate runs post-cluster-provisioning per Phase 3 §7.5.
run: python3 scripts/check_ai600_lula_manifests.py
- name: Assert lula manifest count matches README
run: |
DISK_COUNT=$(ls compliance/lula/lula-validation-*.yaml 2>/dev/null | wc -l | tr -d ' ')
# Count unique filenames referenced in README (grep -c counts lines, not unique names).
README_COUNT=$(grep -o 'lula-validation-[a-z0-9_-]*\.yaml' compliance/lula/README.md | sort -u | wc -l | tr -d ' ')
echo "Manifests on disk (compliance/lula/): $DISK_COUNT"
echo "Unique manifests referenced in README: $README_COUNT"
if [ "$DISK_COUNT" != "$README_COUNT" ]; then
echo "ERROR: Manifest count mismatch. Add new manifests to compliance/lula/README.md or move drafts to compliance/lula/drafts/."
exit 1
fi
echo "OK: manifest count consistent ($DISK_COUNT)"
sbom-generate:
name: "Generate and Validate SBOM"
runs-on: ubuntu-latest
needs: [license-check]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.11"
- name: Install cyclonedx-py
run: pip install cyclonedx-py
- name: Generate SBOM
run: python -m cyclonedx_py environment > sbom.json
- name: Validate SBOM schema
run: |
python -c "
import json
d = json.load(open('sbom.json'))
assert d.get('bomFormat') == 'CycloneDX', \
f'Unexpected bomFormat: {d.get(\"bomFormat\")!r}'
print(f'SBOM validated: {d.get(\"bomFormat\")} v{d.get(\"specVersion\")}')
"
- name: Upload SBOM artifact
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: sbom-${{ github.sha }}
path: sbom.json
retention-days: 90
integration-smoke:
name: "Integration Smoke Tests"
runs-on: ubuntu-latest
# Hard gate: requires live GKE cluster (GOOGLE_CREDENTIALS secret + kubeconfig).
# Move to ci-integration.yml nightly workflow when cluster access is unavailable.
# Fork PRs never have access to repository secrets, so skip the job entirely to
# prevent spurious failures and avoid leaking the absence of secrets in log output.
if: >-
github.event_name != 'pull_request' ||
github.event.pull_request.head.repo.full_name == github.repository
needs: [pytest-logic]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
with:
enable-cache: true
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version-file: "pyproject.toml"
- name: Install dependencies
run: uv sync --all-groups --all-extras
- name: Run integration smoke tests
env:
CAGE_DEPLOYMENT_REGION: US_FED
CAGE_ENV: "ci"
LANGFUSE_POSTURE_DRY_RUN: "true"
OPENAI_API_KEY: "sk-dummy"
MODEL_REASONING: "casperhansen/deepseek-r1-distill-qwen-14b-awq"
MODEL_FAST: "Qwen/Qwen2.5-7B-Instruct"
VLLM_API_KEY: "dummy"
VLLM_GATEWAY_URL: "http://localhost:8081/v1"
REDIS_URL: "redis://localhost:6379/0"
OPA_URL: "http://localhost:8181/v1/data/governance/policy"
LANGFUSE_HOST: "http://localhost:3000"
LANGFUSE_PUBLIC_KEY: "pk-dummy"
LANGFUSE_SECRET_KEY: "sk-dummy"
run: uv run pytest tests/ -m integration --timeout=30 -x -q --ignore=tests/load --ignore=tests/red_team --no-cov
ai600-unit-tests:
name: "AI 600-1 Unit Tests"
runs-on: ubuntu-latest
needs: [license-check]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
with:
enable-cache: true
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version-file: "pyproject.toml"
- name: Install dependencies
run: uv sync --all-groups --all-extras
- name: Run red-team unit tests (mocked, no live services)
env:
CAGE_ENV: "ci"
OPENAI_API_KEY: "sk-dummy"
MODEL_REASONING: "casperhansen/deepseek-r1-distill-qwen-14b-awq"
MODEL_FAST: "Qwen/Qwen2.5-7B-Instruct"
VLLM_API_KEY: "dummy"
VLLM_GATEWAY_URL: "http://localhost:8081/v1"
REDIS_URL: "redis://localhost:6379/0"
OPA_URL: "http://localhost:8181/v1/data/governance/policy"
LANGFUSE_HOST: "http://localhost:3000"
LANGFUSE_PUBLIC_KEY: "pk-dummy"
LANGFUSE_SECRET_KEY: "sk-dummy"
CAGE_DEPLOYMENT_REGION: "APAC_MAS"
GOOGLE_CLOUD_LOCATION: "asia-southeast1"
CAGE_ROUTING_SEAL_SECRET: "dev-only-insecure-placeholder-not-for-production-use"
GOVERNANCE_SALT: "dev-only-insecure-placeholder-not-for-production-use"
run: |
uv run pytest tests/red_team/ -m "red_team and not integration" -v --no-cov
uv run pytest tests/ -m apac_mas -v --no-cov
locust-load-test:
name: "Nightly Load Test (Locust)"
runs-on: ubuntu-latest
# Run only on the nightly schedule — not on every push or pull_request.
if: false # github.event_name == 'schedule'
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
with:
enable-cache: true
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version-file: "pyproject.toml"
- name: Install dependencies
run: uv sync --all-groups --all-extras
- name: Start gateway under test (background)
env:
CAGE_ENV: "ci"
OPA_CACHE_ENABLED: "false"
OPA_URL: "http://localhost:8181/v1/data/trade/governance"
REDIS_URL: "redis://localhost:6379/0"
OPENAI_API_KEY: "sk-dummy"
MODEL_FAST: "Qwen/Qwen2.5-7B-Instruct"
VLLM_API_KEY: "dummy"
VLLM_GATEWAY_URL: "http://localhost:8081/v1"
LANGFUSE_HOST: "http://localhost:3000"
LANGFUSE_PUBLIC_KEY: "pk-dummy"
LANGFUSE_SECRET_KEY: "sk-dummy"
run: |
uv run python -m uvicorn src.gateway.server.mcp_tool_server:app \
--host 0.0.0.0 --port 8080 &
# Wait up to 30 s for the server to become healthy.
for i in $(seq 1 30); do
curl -sf http://localhost:8080/health && break || sleep 1
done
echo "Gateway ready."
- name: Run Locust headless (10 users, 30 s)
run: |
uv run locust -f tests/load/locustfile.py \
--headless -u 10 -r 5 -t 30s \
--host http://localhost:8080 \
--html /tmp/locust-report.html \
--csv /tmp/locust-stats \
--exit-code-on-error 1
- name: Assert p95 latency regression gate
run: |
uv run python scripts/check_locust_baseline.py \
--stats-csv /tmp/locust-stats_stats.csv \
--p95-baseline-ms 2000
- name: Upload Locust report
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: locust-report-${{ github.sha }}
path: /tmp/locust-*.html
retention-days: 30
cbrn-keyword-check:
name: "CBRN Keyword List Validation"
runs-on: ubuntu-latest
needs: [license-check]
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.11"
- name: Validate CBRN keyword list
run: |
python -c "
import json
with open('config/governance_thresholds.json') as f:
cfg = json.load(f)
assert 'tier1_keywords_cbrn' in cfg, 'CBRN keyword list missing from governance_thresholds.json'
assert len(cfg['tier1_keywords_cbrn']) >= 10, \
f'CBRN keyword list too short: {len(cfg[\"tier1_keywords_cbrn\"])} terms (need >= 10)'
assert cfg.get('tier1_keywords_cbrn_enabled') == True, \
'tier1_keywords_cbrn_enabled must be true for US_FED deployment'
print(f'CBRN keywords: {len(cfg[\"tier1_keywords_cbrn\"])} terms — OK')
"
- name: Validate agentic scope statement present
run: |
test -f docs/governance/AGENTIC_SCOPE_STATEMENT.md || \
(echo "ERROR: docs/governance/AGENTIC_SCOPE_STATEMENT.md missing" && exit 1)
echo "Agentic scope statement present — OK"
distributed-cbf-proof:
name: "Distributed CBF Formal Verification"
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- name: Install uv
uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0
with:
enable-cache: true
- name: Set up Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version-file: "pyproject.toml"
- name: Install dependencies
run: uv sync --all-groups --all-extras
- name: Run Distributed CBF Proof
# The proof module verifies safety properties SP-1 through SP-4 for
# N ∈ {2, 3, 4} concurrent agents using BFS state-space enumeration.
# Exit code 0 on success, 1 on safety violation detection.
run: |
uv run python -m proof.distributed_cbf_model
env:
PYTHONPATH: .
- name: Run Distributed CBF Tests
# Pinned state counts verify the model hasn't drifted from documented
# figures. If state space changes, update EXPECTED_STATE_COUNTS in
# proof/distributed_cbf_model.py and document in REVISION_TRACKER.md.
run: |
uv run pytest proof/distributed_cbf_model.py -v --tb=short -o addopts=""