Define roadmap for evidence-based AI change decisions (#6) #35

Workflow file for this run

	name: CI
	on:
	push: {branches: [main]}
	pull_request:
	workflow_call:
	permissions: {}
	concurrency:
	group: ci-${{ github.ref }}
	cancel-in-progress: ${{ github.event_name == 'pull_request' }}
	jobs:
	audit:
	permissions: {contents: read} # lecture du dépôt uniquement
	runs-on: ubuntu-latest
	strategy:
	matrix: {python: ["3.9", "3.13"]}
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with: {persist-credentials: false}
	- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
	with: {python-version: "${{ matrix.python }}"}
	- name: Offline selftest (7 blueprints, zero network)
	run: cd scripts && python forge_dashboards.py --selftest --with-alerts
	- name: Audit harness (4 topologies + regressions)
	# pyyaml is a harness dependency only; the shipped scripts are stdlib
	# and the harness itself asserts that.
	run: \|
	pip install --quiet pyyaml
	python tests/audit_harness.py
	- name: Infrastructure YAML must lint and load
	if: matrix.python == '3.13'
	run: \|
	pip install --quiet yamllint==1.38.0
	yamllint demo/ .github/
	PROM_VER=3.14.0
	BASE=https://github.com/prometheus/prometheus/releases/download
	curl -sL "${BASE}/v${PROM_VER}/prometheus-${PROM_VER}.linux-amd64.tar.gz" -o p.tgz
	echo "f665c6da19eb7ba399c915d30c7d9793c9b417bf8a749b504bc470678631478d p.tgz" \| sha256sum -c -
	tar xzf p.tgz
	./prometheus-${PROM_VER}.linux-amd64/promtool check config demo/prometheus.yml
	cd scripts && python forge_dashboards.py --selftest >/dev/null
	../prometheus-${PROM_VER}.linux-amd64/promtool check rules \
	selftest_output/prometheus_rules_llmops.yml
	python - <<'EOF'
	import yaml
	crd = yaml.safe_load(open("selftest_output/prometheusrule_llmops.yaml"))
	yaml.safe_dump({"groups": crd["spec"]["groups"]}, open("/tmp/crd.yml", "w"))
	EOF
	../prometheus-${PROM_VER}.linux-amd64/promtool check rules /tmp/crd.yml

	- name: Financial sources and recorded costs preserve scope and values
	if: matrix.python == '3.13'
	run: \|
	python tests/recorded_cost_invariants.py --promtool ./prometheus-3.14.0.linux-amd64/promtool
	python tests/financial_source_invariants.py --promtool ./prometheus-3.14.0.linux-amd64/promtool
	python tests/financial_coverage_invariants.py

	- name: Quality statistics retain their meaning in English and French
	if: matrix.python == '3.13'
	run: python tests/quality_stat_invariants.py --promtool ./prometheus-3.14.0.linux-amd64/promtool

	- name: Deployment declarations preserve scope and remain plain text
	run: python tests/deployment_inventory_invariants.py

	- name: Budget alerts preserve safe lifecycle state
	run: python tests/budget_alert_lifecycle_invariants.py

	- name: Workflows are code, lint them too
	if: matrix.python == '3.13'
	run: \|
	pip install --quiet zizmor==1.29.0
	zizmor --no-online-audits --persona=regular .github/workflows/
	- name: Package builds reproducibly and matches sources
	run: \|
	python tools/package.py
	python tools/package.py --verify
	python tests/supply_chain_check.py
	- name: Registry JSON is valid & priced models have sources
	run: \|
	python - <<'PY'
	import json
	import sys
	sys.path.insert(0, "scripts")
	from pricing_sources import validate_registry

	r = json.load(open("references/model_registry.json"))
	validate_registry(r)
	assert r["models"], "registry incomplete"
	print(len(r["models"]), "models OK")
	PY

	live-queries:
	permissions: {contents: read} # lecture du dépôt uniquement
	runs-on: ubuntu-latest
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with: {persist-credentials: false}
	- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
	with: {python-version: "3.12"}
	- name: Start Prometheus + synthetic emitter (pinned images, no downloads)
	run: \|
	docker compose -f demo/docker-compose.yml up -d --wait emitter prometheus
	- name: Every generated query must return data
	# No fixed sleep: the script waits until Prometheus has ingested traffic.
	run: \|
	python tests/live_query_check.py --prometheus http://localhost:9090 \
	--wait-for-data 180
	- name: Load the generated cost recording rules and let them evaluate
	run: \|
	python tests/live_query_check.py --prometheus http://localhost:9090 \
	--wait-for-data 180 --out-dir /tmp/lf >/dev/null
	cp /tmp/lf/prometheus_rules_llmops.yml demo/rules/
	docker compose -f demo/docker-compose.yml restart prometheus
	- name: Values must be internally consistent
	# Pas de sleep fixe : le script attend que les regles se materialisent.
	run: \|
	python tests/value_invariants.py --prometheus http://localhost:9090 \
	--wait-for-rules 240
	- name: Diagnose on failure (compose ps + container logs)
	if: failure()
	run: \|
	docker compose -f demo/docker-compose.yml ps -a
	docker compose -f demo/docker-compose.yml logs --tail=150

	windows-portability:
	name: Windows portability (Python 3.13)
	permissions: {contents: read}
	runs-on: windows-latest
	env:
	PYTHONUTF8: "1"
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with: {persist-credentials: false}
	- uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
	with: {python-version: "3.13"}
	- name: Install harness dependency
	run: python -m pip install --quiet pyyaml==6.0.2
	- name: Offline selftest
	run: python scripts/forge_dashboards.py --selftest --with-alerts
	- name: Audit harness
	run: python tests/audit_harness.py
	- name: Package and SBOM are portable and reproducible
	run: \|
	python tools/package.py
	python tools/sbom.py dist/grafana-llmops-forge.skill
	python tools/package.py --verify
	python tests/supply_chain_check.py
	- name: Package verification is safe under Windows CP1252
	env:
	PYTHONUTF8: "0"
	PYTHONIOENCODING: cp1252
	run: \|
	python tools/package.py --verify
	python tools/sbom.py dist/grafana-llmops-forge.skill --verify

	e2e:
	name: E2E Grafana ${{ matrix.version }} (${{ matrix.support }})
	permissions: {contents: read} # lecture du dépôt uniquement
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	include:
	- version: "9.5.21"
	support: legacy-compatibility-only
	grafana: grafana/grafana:9.5.21@sha256:ec106c7d446c88377f9d6c4cd363361b5846c361a28f5669d1b5e92926d94891
	- version: "10.4.19"
	support: legacy-compatibility-only
	grafana: grafana/grafana:10.4.19@sha256:a9043254ba16fb10945cc27333963dfd08eccbb43b51f1222d831cc564e3a1f4
	- version: "11.6.16"
	support: legacy-compatibility-only
	grafana: grafana/grafana:11.6.16@sha256:d67af92050b8d93b393dc741864752a69c9da1ffa39c1bb9af49ad5d9e47d2c3
	- version: "12.4.9"
	support: supported
	grafana: grafana/grafana:12.4.9@sha256:9b58461280b4d2992d4399823c9427d0fcf5f0fd7f376c93f2dea876158b867b
	- version: "13.2.0"
	support: supported
	grafana: grafana/grafana:13.2.0@sha256:3fd54ae1214669f8355f065ec9f6445d5279a3d77095ab048ca045685272429b
	env:
	GRAFANA_IMAGE: ${{ matrix.grafana }}
	steps:
	- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
	with: {persist-credentials: false}
	- name: Boot demo stack (Grafana + Prometheus + synthetic LLM metrics)
	env: {GF_PLUGINS: ""} # renderer not needed here; skips a network install at boot
	run: docker compose -f demo/docker-compose.yml up -d --wait
	- name: Wait for the first scrape
	# Pas de sleep devine, ici non plus. `up --wait` a deja bloque sur les
	# healthchecks ; il reste a laisser UN scrape complet atterrir, car la
	# decouverte matche des noms de metriques et Prometheus ingere tout
	# l'endpoint d'un coup. On attend donc la serie, pas une duree : sortie
	# des qu'elle est la, echec franc et date si elle ne vient jamais.
	run: \|
	MAX=90 # une seule source pour le budget, boucle et message compris
	for i in $(seq 1 "$MAX"); do
	if curl -sf http://localhost:9090/api/v1/label/__name__/values \
	\| grep -q '"gen_ai_'; then
	echo "series gen_ai visibles apres ${i}s"; exit 0
	fi
	sleep 1
	done
	echo "::error::aucune serie gen_ai apres ${MAX}s (emitter ou scrape KO)"
	exit 1
	- name: Discover, forge and deploy for real
	env: {GRAFANA_URL: "http://localhost:3000", GRAFANA_USER: admin, GRAFANA_PASSWORD: admin}
	run: \|
	python scripts/discover.py --out cap.json
	python - <<'PY'
	import json
	cap = json.load(open("cap.json"))
	print("dialects:", sorted({d for s in cap["signals"].values() for d in s}))
	PY
	python scripts/forge_dashboards.py --capability cap.json \
	--blueprints auto --deploy --with-alerts --export-portable
	- name: Assert dashboards are live and queryable
	env: {GRAFANA_URL: "http://localhost:3000"}
	run: \|
	python - <<'PY'
	import base64, json, urllib.request
	auth = base64.b64encode(b"admin:admin").decode()
	def get(p):
	r = urllib.request.Request(
	"http://localhost:3000" + p,
	headers={"Authorization": "Basic " + auth})
	return json.load(urllib.request.urlopen(r))
	ds = get("/api/search?tag=llmops-forge")
	assert len(ds) >= 5, f"expected >=5 dashboards, got {len(ds)}"
	for d in ds:
	full = get("/api/dashboards/uid/"+d["uid"])
	assert full["dashboard"]["panels"], d["title"]
	print(f"{len(ds)} dashboards live:", [d["title"] for d in ds])
	PY
	- name: Budget coverage changes pause the scoped live rule and preserve that pause
	env: {GRAFANA_URL: "http://localhost:3000", GRAFANA_USER: admin, GRAFANA_PASSWORD: admin}
	run: \|
	python - <<'PY'
	import base64, copy, json, pathlib, subprocess, sys, urllib.request

	auth = base64.b64encode(b"admin:admin").decode()
	def get(path):
	request = urllib.request.Request(
	"http://localhost:3000" + path,
	headers={"Authorization": "Basic " + auth})
	with urllib.request.urlopen(request, timeout=15) as response:
	return json.load(response)

	endpoint = "/api/v1/provisioning/alert-rules"
	initial_rules = {r["uid"]: r for r in get(endpoint)}
	assert initial_rules, "the initial deployment must provide unrelated rules"
	initial_folders = {r["folderUID"] for r in initial_rules.values()}
	folders_before = {uid: get("/api/folders/" + uid) for uid in initial_folders}
	org_id = get("/api/org")["id"]
	root = pathlib.Path("budget-lifecycle-e2e")
	root.mkdir()
	cap = json.load(open("cap.json"))
	ds_uid = next(uid for uid, signals in cap["signals"].items() if "otel_genai" in signals)
	entry = copy.deepcopy(cap["signals"][ds_uid]["otel_genai"])
	# Keep the token histogram and sum; omit request/latency signals so this
	# dedicated fixture generates only the budget rule, with real datasource UIDs.
	entry["metric_names"] = [name for name in entry["metric_names"]
	if "token_usage" in name and name.endswith(("_bucket", "_sum"))]
	assert len(entry["metric_names"]) == 2, entry["metric_names"]
	entry["providers_seen"] = ["openai"]
	cap["signals"] = {ds_uid: {"otel_genai": entry}}
	cap["datasources"]["prometheus"] = [d for d in cap["datasources"]["prometheus"]
	if d["uid"] == ds_uid]

	def deploy(stage, models):
	entry["models_seen"] = models
	entry["discovery_coverage"] = {
	"scope": "backend_returned_values", "local_truncation": False,
	"backend_completeness": "unknown",
	"counts": {key: len(entry[key])
	for key in ("metric_names", "models_seen", "providers_seen")}}
	capability = root / (stage + ".json")
	capability.write_text(json.dumps(cap), encoding="utf-8")
	out = root / stage
	subprocess.run([
	sys.executable, "scripts/forge_dashboards.py", "--capability", str(capability),
	"--blueprints", "finops", "--cost-mode", "inline", "--deploy", "--with-alerts",
	"--folder", "Forge CI budget lifecycle", "--uid-scope", "ci-budget-lifecycle",
	"--out-dir", str(out)], check=True)
	manifest = json.loads((out / "deploy_manifest.json").read_text(encoding="utf-8"))
	assert manifest["deployment_status"] == "success" and not manifest["errors"], stage
	assert manifest["deployed"] and manifest["uid_scope"] == "ci-budget-lifecycle", stage
	return manifest

	def assert_unrelated_unchanged(budget_uid):
	current = {r["uid"]: r for r in get(endpoint)}
	assert set(current) == set(initial_rules) \| {budget_uid}, "unexpected rule set change"
	assert {uid: current[uid] for uid in initial_rules} == initial_rules, "unrelated rules changed"
	assert {uid: get("/api/folders/" + uid) for uid in initial_folders} == folders_before

	complete = ["gpt-5.4"]
	partial = complete + ["forge-ci-unpriced-budget-model"]
	first = deploy("eligible", complete)
	assert first["financial_source"]["coverage"]["budget_eligible"] is True
	assert len(first["alerts"]) == 1, "fixture must isolate the budget rule"
	budget_uid = first["alerts"][0]["uid"]
	assert "llm-daily-budget" in budget_uid and budget_uid not in initial_rules
	folder_uid = first["folder_uid"]
	assert folder_uid not in initial_folders
	live = get(endpoint + "/" + budget_uid)
	assert live["uid"] == budget_uid and live["ruleGroup"] == "llmops-slo"
	assert live["orgID"] == org_id and live["folderUID"] == folder_uid
	assert live["labels"]["origin"] == "llmops-forge" and live["labels"]["severity"] == "warning"
	assert next(query for query in live["data"] if query["refId"] == "A")["datasourceUid"] == ds_uid
	assert live["isPaused"] is False, "eligible budget must initially be active"
	assert_unrelated_unchanged(budget_uid)

	for stage, result in (("partial", "paused"), ("partial-again", "already_paused")):
	manifest = deploy(stage, partial)
	assert get(endpoint + "/" + budget_uid)["isPaused"] is True, stage
	coverage = manifest["financial_source"]["coverage"]
	assert coverage["status"] == "partial_prices" and coverage["budget_eligible"] is False
	assert len(manifest["alerts"]) == 1, "only the budget maintenance operation is expected"
	maintenance = manifest["alerts"][0]
	assert maintenance["uid"] == budget_uid and maintenance["action"] == "pause"
	assert maintenance["status"] == "succeeded" and maintenance["result"] == result
	assert manifest["resources"]["alerts"] == {
	"requested": 1, "succeeded": 1, "failed": 0, "skipped": 0}
	state = manifest["budget_alert"]
	assert state["uid"] == budget_uid and state["folder_uid"] == folder_uid
	assert state["org_id"] == org_id and state["action"] == "pause"
	assert state["result"] == result and state["is_paused"] is True, state
	assert state["reason"] and state["pause_policy"] == "preserve_existing_pause"
	assert_unrelated_unchanged(budget_uid)

	restored = deploy("eligible-again", complete)
	assert restored["financial_source"]["coverage"]["budget_eligible"] is True
	assert [a["uid"] for a in restored["alerts"]] == [budget_uid]
	assert get(endpoint + "/" + budget_uid)["isPaused"] is True, "restoring coverage must not unpause"
	state = restored["budget_alert"]
	assert state["uid"] == budget_uid and state["action"] == "upsert"
	assert state["result"] == "upserted_paused" and state["is_paused"] is True, state
	assert state["pause_policy"] == "preserve_existing_pause"
	assert_unrelated_unchanged(budget_uid)
	print("Budget lifecycle PASS: active -> paused -> already_paused -> still paused; unrelated rules intact")
	PY
	- name: Diagnose on failure (compose ps + container logs)
	if: failure()
	run: \|
	docker compose -f demo/docker-compose.yml ps -a
	docker compose -f demo/docker-compose.yml logs --tail=150
	- name: Teardown
	if: always()
	run: docker compose -f demo/docker-compose.yml down -v

Navigation Menu

Define roadmap for evidence-based AI change decisions (#6) #35

Workflow file

Define roadmap for evidence-based AI change decisions (#6) #35

Uh oh!

Workflow file for this run