fix(alerting): pause budgets with incomplete cost coverage

Author: alebgl77

.github/workflows/ci.yml

@@ -61,6 +61,9 @@ jobs:
       - name: Deployment declarations preserve scope and remain plain text
         run: python tests/deployment_inventory_invariants.py
 
+      - name: Budget alerts preserve safe lifecycle state
+        run: python tests/budget_alert_lifecycle_invariants.py
+
       - name: Workflows are code, lint them too
         if: matrix.python == '3.13'
         run: |
@@ -226,6 +229,113 @@ jobs:
               assert full["dashboard"]["panels"], d["title"]
           print(f"{len(ds)} dashboards live:", [d["title"] for d in ds])
           PY
+      - name: Budget coverage changes pause the scoped live rule and preserve that pause
+        env: {GRAFANA_URL: "http://localhost:3000", GRAFANA_USER: admin, GRAFANA_PASSWORD: admin}
+        run: |
+          python - <<'PY'
+          import base64, copy, json, pathlib, subprocess, sys, urllib.request
+
+          auth = base64.b64encode(b"admin:admin").decode()
+          def get(path):
+              request = urllib.request.Request(
+                  "http://localhost:3000" + path,
+                  headers={"Authorization": "Basic " + auth})
+              with urllib.request.urlopen(request, timeout=15) as response:
+                  return json.load(response)
+
+          endpoint = "/api/v1/provisioning/alert-rules"
+          initial_rules = {r["uid"]: r for r in get(endpoint)}
+          assert initial_rules, "the initial deployment must provide unrelated rules"
+          initial_folders = {r["folderUID"] for r in initial_rules.values()}
+          folders_before = {uid: get("/api/folders/" + uid) for uid in initial_folders}
+          org_id = get("/api/org")["id"]
+          root = pathlib.Path("budget-lifecycle-e2e")
+          root.mkdir()
+          cap = json.load(open("cap.json"))
+          ds_uid = next(uid for uid, signals in cap["signals"].items() if "otel_genai" in signals)
+          entry = copy.deepcopy(cap["signals"][ds_uid]["otel_genai"])
+          # Keep the token histogram and sum; omit request/latency signals so this
+          # dedicated fixture generates only the budget rule, with real datasource UIDs.
+          entry["metric_names"] = [name for name in entry["metric_names"]
+                                   if "token_usage" in name and name.endswith(("_bucket", "_sum"))]
+          assert len(entry["metric_names"]) == 2, entry["metric_names"]
+          entry["providers_seen"] = ["openai"]
+          cap["signals"] = {ds_uid: {"otel_genai": entry}}
+          cap["datasources"]["prometheus"] = [d for d in cap["datasources"]["prometheus"]
+                                               if d["uid"] == ds_uid]
+
+          def deploy(stage, models):
+              entry["models_seen"] = models
+              entry["discovery_coverage"] = {
+                  "scope": "backend_returned_values", "local_truncation": False,
+                  "backend_completeness": "unknown",
+                  "counts": {key: len(entry[key])
+                             for key in ("metric_names", "models_seen", "providers_seen")}}
+              capability = root / (stage + ".json")
+              capability.write_text(json.dumps(cap), encoding="utf-8")
+              out = root / stage
+              subprocess.run([
+                  sys.executable, "scripts/forge_dashboards.py", "--capability", str(capability),
+                  "--blueprints", "finops", "--cost-mode", "inline", "--deploy", "--with-alerts",
+                  "--folder", "Forge CI budget lifecycle", "--uid-scope", "ci-budget-lifecycle",
+                  "--out-dir", str(out)], check=True)
+              manifest = json.loads((out / "deploy_manifest.json").read_text(encoding="utf-8"))
+              assert manifest["deployment_status"] == "success" and not manifest["errors"], stage
+              assert manifest["deployed"] and manifest["uid_scope"] == "ci-budget-lifecycle", stage
+              return manifest
+
+          def assert_unrelated_unchanged(budget_uid):
+              current = {r["uid"]: r for r in get(endpoint)}
+              assert set(current) == set(initial_rules) | {budget_uid}, "unexpected rule set change"
+              assert {uid: current[uid] for uid in initial_rules} == initial_rules, "unrelated rules changed"
+              assert {uid: get("/api/folders/" + uid) for uid in initial_folders} == folders_before
+
+          complete = ["gpt-5.4"]
+          partial = complete + ["forge-ci-unpriced-budget-model"]
+          first = deploy("eligible", complete)
+          assert first["financial_source"]["coverage"]["budget_eligible"] is True
+          assert len(first["alerts"]) == 1, "fixture must isolate the budget rule"
+          budget_uid = first["alerts"][0]["uid"]
+          assert "llm-daily-budget" in budget_uid and budget_uid not in initial_rules
+          folder_uid = first["folder_uid"]
+          assert folder_uid not in initial_folders
+          live = get(endpoint + "/" + budget_uid)
+          assert live["uid"] == budget_uid and live["ruleGroup"] == "llmops-slo"
+          assert live["orgID"] == org_id and live["folderUID"] == folder_uid
+          assert live["labels"]["origin"] == "llmops-forge" and live["labels"]["severity"] == "warning"
+          assert next(query for query in live["data"] if query["refId"] == "A")["datasourceUid"] == ds_uid
+          assert live["isPaused"] is False, "eligible budget must initially be active"
+          assert_unrelated_unchanged(budget_uid)
+
+          for stage, result in (("partial", "paused"), ("partial-again", "already_paused")):
+              manifest = deploy(stage, partial)
+              assert get(endpoint + "/" + budget_uid)["isPaused"] is True, stage
+              coverage = manifest["financial_source"]["coverage"]
+              assert coverage["status"] == "partial_prices" and coverage["budget_eligible"] is False
+              assert len(manifest["alerts"]) == 1, "only the budget maintenance operation is expected"
+              maintenance = manifest["alerts"][0]
+              assert maintenance["uid"] == budget_uid and maintenance["action"] == "pause"
+              assert maintenance["status"] == "succeeded" and maintenance["result"] == result
+              assert manifest["resources"]["alerts"] == {
+                  "requested": 1, "succeeded": 1, "failed": 0, "skipped": 0}
+              state = manifest["budget_alert"]
+              assert state["uid"] == budget_uid and state["folder_uid"] == folder_uid
+              assert state["org_id"] == org_id and state["action"] == "pause"
+              assert state["result"] == result and state["is_paused"] is True, state
+              assert state["reason"] and state["pause_policy"] == "preserve_existing_pause"
+              assert_unrelated_unchanged(budget_uid)
+
+          restored = deploy("eligible-again", complete)
+          assert restored["financial_source"]["coverage"]["budget_eligible"] is True
+          assert [a["uid"] for a in restored["alerts"]] == [budget_uid]
+          assert get(endpoint + "/" + budget_uid)["isPaused"] is True, "restoring coverage must not unpause"
+          state = restored["budget_alert"]
+          assert state["uid"] == budget_uid and state["action"] == "upsert"
+          assert state["result"] == "upserted_paused" and state["is_paused"] is True, state
+          assert state["pause_policy"] == "preserve_existing_pause"
+          assert_unrelated_unchanged(budget_uid)
+          print("Budget lifecycle PASS: active -> paused -> already_paused -> still paused; unrelated rules intact")
+          PY
       - name: Diagnose on failure (compose ps + container logs)
         if: failure()
         run: |

README.md

@@ -92,6 +92,8 @@ Discovery preserves every metric name, model and provider value returned by the
 
 Inline composition is capped at **40 models with usable prices**. Above that limit, no monetary subtotal or budget is generated: select the financial datasource with `--datasource`, generate and load its recording rules, then repeat discovery and forge. All priced models supplied to the rule generator are retained, including beyond 60 models. Recorded queries are shorter, but execution cost still depends on series count and time window. Native and recorded sources retain unverified upstream coverage and do not depend on registry coverage.
 
+With `--deploy --with-alerts`, an ineligible budget also pauses an existing Forge budget in the resolved organization and folder. Forge checks its deterministic UID, rule group and ownership labels before updating only `isPaused`; an absent budget is not created. The manifest records `budget_alert.action`, `result` (`paused`, `already_paused`, `absent` or `failed`) and the coverage reason. Alert resource counts include this maintenance operation, including confirmed absence or an already paused rule. A collision or unconfirmed HTTP response makes the deployment partial/failed. Generation, dry-run and deployments without `--with-alerts` do not perform this check. When coverage becomes eligible again, Forge updates the budget while preserving its existing pause (`upserted_paused`): review the coverage and resume it explicitly in Grafana. Other SLO pause behavior is unchanged.
+
 ### Provider origin and declared deployment locations
 
 The registry field and Prometheus label `region` remain compatible and describe **provider origin**. They establish neither processing nor storage location. Governance always includes a separate deployment panel: locations default to **unknown**. Optionally pass `--deployment-inventory ../instance/deployment_inventory.json`, keeping real inventory outside the packaged skill. A US-origin model can have declared processing in France; both facts are shown separately.

scripts/forge_dashboards.py

@@ -2015,6 +2015,13 @@ def main() -> int:
     folder_uid = det_uid(args.folder, "fold", args.uid_scope)
     alert_rules = (build_alerts(ctx, folder_uid, args.daily_budget, args.slo_target)
                    if args.with_alerts else [])
+    budget_uid = det_uid("llm-daily-budget", "alr", args.uid_scope)
+    budget_generated = any(rule["uid"] == budget_uid for rule in alert_rules)
+    budget_reason = (None if budget_generated else
+                     (ctx.cost_source.coverage["budget_omission_reason"] if ctx.cost_source else None)
+                     or "No usable financial source or budget cost expression.")
+    pause_budget = operation == "deploy" and args.with_alerts and not budget_generated
+    budget_action = ("upsert" if budget_generated else "pause") if args.with_alerts else "not_requested"
     manifest = {
         "schema": "grafana-llmops-forge/deployment-manifest",
         "version": 2,
@@ -2029,12 +2036,20 @@ def main() -> int:
         "uid_scope": args.uid_scope,
         "financial_source": ctx.financial_manifest(),
         "deployment_inventory": ctx.deployment_manifest(),
+        "budget_alert": {
+            "uid": budget_uid, "org_id": ctx.org_id, "folder_uid": folder_uid,
+            "action": budget_action,
+            "result": ("pending" if operation == "deploy" else "not_executed")
+                      if args.with_alerts else "not_requested",
+            "reason": budget_reason if args.with_alerts else None,
+            "is_paused": None, "pause_policy": "preserve_existing_pause",
+        },
         "resources": {
             "folder": {"requested": 1 if operation == "deploy" else 0,
                        "succeeded": 0, "failed": 0, "skipped": 0},
             "dashboards": {"requested": len(boards), "succeeded": 0,
                            "failed": 0, "skipped": 0},
-            "alerts": {"requested": len(alert_rules), "succeeded": 0,
+            "alerts": {"requested": len(alert_rules) + int(pause_budget), "succeeded": 0,
                        "failed": 0, "skipped": 0},
         },
         "errors": [],
@@ -2059,6 +2074,11 @@ def main() -> int:
         {"uid": rule["uid"], "title": rule["title"],
          "status": "pending" if operation == "deploy" else "succeeded"}
         for rule in alert_rules]
+    if pause_budget:
+        # This is a scoped maintenance operation, not a generated replacement rule.
+        manifest["alerts"].append(
+            {"uid": budget_uid, "title": "LLM financial budget", "action": "pause",
+             "status": "pending", "reason": budget_reason})
     if operation == "generate":
         manifest["resources"]["dashboards"]["succeeded"] = len(boards)
         manifest["resources"]["alerts"]["succeeded"] = len(alert_rules)
@@ -2083,7 +2103,9 @@ def _perm_hint(op, err):
         except (GrafanaError, SystemExit) as e:
             manifest["resources"]["folder"]["failed"] = 1
             manifest["resources"]["dashboards"]["skipped"] = len(boards)
-            manifest["resources"]["alerts"]["skipped"] = len(alert_rules)
+            manifest["resources"]["alerts"]["skipped"] = len(manifest["alerts"])
+            if args.with_alerts:
+                manifest["budget_alert"]["result"] = "not_executed"
             for entry in manifest["dashboards"] + manifest["alerts"]:
                 entry["status"] = "skipped"
             manifest["errors"].append(
@@ -2098,6 +2120,7 @@ def _perm_hint(op, err):
         else:
             actual_folder_uid = folder.get("uid") or folder_uid
             manifest["folder_uid"] = actual_folder_uid
+            manifest["budget_alert"]["folder_uid"] = actual_folder_uid
             manifest["resources"]["folder"]["succeeded"] = 1
             manifest["deployed"] = True
             print(f"\nFolder '{folder.get('title')}' (uid {actual_folder_uid})")
@@ -2128,18 +2151,62 @@ def _perm_hint(op, err):
                 print(f"\n[partial] {ds_stats['succeeded']}/{ds_stats['requested']} "
                       "dashboards deployed. Re-running after fixing the role is safe: "
                       "deterministic UIDs make it an update.", file=sys.stderr)
-            if alert_rules and not client.contact_points():
-                print("  [warn] no contact point configured: alerts will fire with no "
-                      "recipient (Alerting -> Contact points).")
+            if pause_budget:
+                expected_budget = {
+                    "uid": budget_uid, "orgID": ctx.org_id, "folderUID": actual_folder_uid,
+                    "ruleGroup": "llmops-slo",
+                    "labels": {"origin": "llmops-forge",
+                               "llmops_rule_identity": alert_logical_identity("llm-daily-budget")},
+                }
+                entry = manifest["alerts"][-1]
+                try:
+                    result = client.pause_budget_alert_rule(expected_budget)
+                except (Exception, SystemExit) as e:
+                    manifest["budget_alert"]["result"] = "failed"
+                    entry.update(status="failed", result="failed", error=str(e))
+                    manifest["resources"]["alerts"]["failed"] += 1
+                    manifest["errors"].append(
+                        {"resource_type": "alert", "identifier": "LLM financial budget",
+                         "uid": budget_uid, "operation": "pause",
+                         "status": getattr(e, "status", 0), "message": _perm_hint("alert", e)})
+                    print(f"  [fail] budget pause was not confirmed: {_perm_hint('alert', e)}",
+                          file=sys.stderr)
+                else:
+                    manifest["budget_alert"].update(
+                        result=result, is_paused=True if result != "absent" else None)
+                    entry.update(status="succeeded", result=result)
+                    manifest["resources"]["alerts"]["succeeded"] += 1
+                    print(f"  [ok] budget: {result}; {budget_reason}")
+            if alert_rules:
+                try:
+                    contacts = client.contact_points()
+                except (GrafanaError, SystemExit) as e:
+                    # This informational lookup must not discard a deployment failure manifest.
+                    print(f"  [warn] contact point availability could not be checked: {e}",
+                          file=sys.stderr)
+                else:
+                    if not contacts:
+                        print("  [warn] no contact point configured: alerts will fire with no "
+                              "recipient (Alerting -> Contact points).")
             for i, rule in enumerate(alert_rules):
                 # Folder UID is part of the alert body and must match the resolved folder.
                 rule["folderUID"] = actual_folder_uid
                 try:
-                    client.upsert_alert_rule(rule)
+                    result = client.upsert_alert_rule(rule)
+                    if rule["uid"] == budget_uid:
+                        paused = result.get("isPaused") if isinstance(result, dict) else None
+                        manifest["budget_alert"].update(
+                            result="upserted_paused" if paused is True else "upserted",
+                            is_paused=paused)
+                        if paused is True:
+                            print("  [info] budget remains paused; review coverage and resume "
+                                  "it explicitly in Grafana when appropriate.")
                     manifest["resources"]["alerts"]["succeeded"] += 1
                     manifest["alerts"][i]["status"] = "succeeded"
                     print(f"  [ok] alert: {rule['title']}")
-                except Exception as e:  # droits alerting et transports hétérogènes
+                except (Exception, SystemExit) as e:  # droits alerting et transports hétérogènes
+                    if rule["uid"] == budget_uid:
+                        manifest["budget_alert"]["result"] = "failed"
                     manifest["resources"]["alerts"]["failed"] += 1
                     manifest["alerts"][i]["status"] = "failed"
                     manifest["alerts"][i]["error"] = str(e)