docs: publish product roadmap #27
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| on: | |
| push: {branches: [main]} | |
| pull_request: | |
| workflow_call: | |
| permissions: {} | |
| concurrency: | |
| group: ci-${{ github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' }} | |
| jobs: | |
| audit: | |
| permissions: {contents: read} # lecture du dépôt uniquement | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: {python: ["3.9", "3.13"]} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: {persist-credentials: false} | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: {python-version: "${{ matrix.python }}"} | |
| - name: Offline selftest (7 blueprints, zero network) | |
| run: cd scripts && python forge_dashboards.py --selftest --with-alerts | |
| - name: Audit harness (4 topologies + regressions) | |
| # pyyaml is a harness dependency only; the shipped scripts are stdlib | |
| # and the harness itself asserts that. | |
| run: | | |
| pip install --quiet pyyaml | |
| python tests/audit_harness.py | |
| - name: Infrastructure YAML must lint and load | |
| if: matrix.python == '3.13' | |
| run: | | |
| pip install --quiet yamllint==1.38.0 | |
| yamllint demo/ .github/ | |
| PROM_VER=3.14.0 | |
| BASE=https://github.com/prometheus/prometheus/releases/download | |
| curl -sL "${BASE}/v${PROM_VER}/prometheus-${PROM_VER}.linux-amd64.tar.gz" -o p.tgz | |
| echo "f665c6da19eb7ba399c915d30c7d9793c9b417bf8a749b504bc470678631478d p.tgz" | sha256sum -c - | |
| tar xzf p.tgz | |
| ./prometheus-${PROM_VER}.linux-amd64/promtool check config demo/prometheus.yml | |
| cd scripts && python forge_dashboards.py --selftest >/dev/null | |
| ../prometheus-${PROM_VER}.linux-amd64/promtool check rules \ | |
| selftest_output/prometheus_rules_llmops.yml | |
| python - <<'EOF' | |
| import yaml | |
| crd = yaml.safe_load(open("selftest_output/prometheusrule_llmops.yaml")) | |
| yaml.safe_dump({"groups": crd["spec"]["groups"]}, open("/tmp/crd.yml", "w")) | |
| EOF | |
| ../prometheus-${PROM_VER}.linux-amd64/promtool check rules /tmp/crd.yml | |
| - name: Workflows are code, lint them too | |
| if: matrix.python == '3.13' | |
| run: | | |
| pip install --quiet zizmor==1.29.0 | |
| zizmor --no-online-audits --persona=regular .github/workflows/ | |
| - name: Package builds reproducibly and matches sources | |
| run: | | |
| python tools/package.py | |
| python tools/package.py --verify | |
| python tests/supply_chain_check.py | |
| - name: Registry JSON is valid & priced models have sources | |
| run: | | |
| python - <<'PY' | |
| import json | |
| import sys | |
| sys.path.insert(0, "scripts") | |
| from pricing_sources import validate_registry | |
| r = json.load(open("references/model_registry.json")) | |
| validate_registry(r) | |
| assert r["models"], "registry incomplete" | |
| print(len(r["models"]), "models OK") | |
| PY | |
| live-queries: | |
| permissions: {contents: read} # lecture du dépôt uniquement | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: {persist-credentials: false} | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: {python-version: "3.12"} | |
| - name: Start Prometheus + synthetic emitter (pinned images, no downloads) | |
| run: | | |
| docker compose -f demo/docker-compose.yml up -d --wait emitter prometheus | |
| - name: Every generated query must return data | |
| # No fixed sleep: the script waits until Prometheus has ingested traffic. | |
| run: | | |
| python tests/live_query_check.py --prometheus http://localhost:9090 \ | |
| --wait-for-data 180 | |
| - name: Load the generated cost recording rules and let them evaluate | |
| run: | | |
| python tests/live_query_check.py --prometheus http://localhost:9090 \ | |
| --wait-for-data 180 --out-dir /tmp/lf >/dev/null | |
| cp /tmp/lf/prometheus_rules_llmops.yml demo/rules/ | |
| docker compose -f demo/docker-compose.yml restart prometheus | |
| - name: Values must be internally consistent | |
| # Pas de sleep fixe : le script attend que les regles se materialisent. | |
| run: | | |
| python tests/value_invariants.py --prometheus http://localhost:9090 \ | |
| --wait-for-rules 240 | |
| - name: Diagnose on failure (compose ps + container logs) | |
| if: failure() | |
| run: | | |
| docker compose -f demo/docker-compose.yml ps -a | |
| docker compose -f demo/docker-compose.yml logs --tail=150 | |
| windows-portability: | |
| name: Windows portability (Python 3.13) | |
| permissions: {contents: read} | |
| runs-on: windows-latest | |
| env: | |
| PYTHONUTF8: "1" | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: {persist-credentials: false} | |
| - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 | |
| with: {python-version: "3.13"} | |
| - name: Install harness dependency | |
| run: python -m pip install --quiet pyyaml==6.0.2 | |
| - name: Offline selftest | |
| run: python scripts/forge_dashboards.py --selftest --with-alerts | |
| - name: Audit harness | |
| run: python tests/audit_harness.py | |
| - name: Package and SBOM are portable and reproducible | |
| run: | | |
| python tools/package.py | |
| python tools/sbom.py dist/grafana-llmops-forge.skill | |
| python tools/package.py --verify | |
| python tests/supply_chain_check.py | |
| - name: Package verification is safe under Windows CP1252 | |
| env: | |
| PYTHONUTF8: "0" | |
| PYTHONIOENCODING: cp1252 | |
| run: | | |
| python tools/package.py --verify | |
| python tools/sbom.py dist/grafana-llmops-forge.skill --verify | |
| e2e: | |
| name: E2E Grafana ${{ matrix.version }} (${{ matrix.support }}) | |
| permissions: {contents: read} # lecture du dépôt uniquement | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - version: "9.5.21" | |
| support: legacy-compatibility-only | |
| grafana: grafana/grafana:9.5.21@sha256:ec106c7d446c88377f9d6c4cd363361b5846c361a28f5669d1b5e92926d94891 | |
| - version: "10.4.19" | |
| support: legacy-compatibility-only | |
| grafana: grafana/grafana:10.4.19@sha256:a9043254ba16fb10945cc27333963dfd08eccbb43b51f1222d831cc564e3a1f4 | |
| - version: "11.6.16" | |
| support: legacy-compatibility-only | |
| grafana: grafana/grafana:11.6.16@sha256:d67af92050b8d93b393dc741864752a69c9da1ffa39c1bb9af49ad5d9e47d2c3 | |
| - version: "12.4.9" | |
| support: supported | |
| grafana: grafana/grafana:12.4.9@sha256:9b58461280b4d2992d4399823c9427d0fcf5f0fd7f376c93f2dea876158b867b | |
| - version: "13.2.0" | |
| support: supported | |
| grafana: grafana/grafana:13.2.0@sha256:3fd54ae1214669f8355f065ec9f6445d5279a3d77095ab048ca045685272429b | |
| env: | |
| GRAFANA_IMAGE: ${{ matrix.grafana }} | |
| steps: | |
| - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 | |
| with: {persist-credentials: false} | |
| - name: Boot demo stack (Grafana + Prometheus + synthetic LLM metrics) | |
| env: {GF_PLUGINS: ""} # renderer not needed here; skips a network install at boot | |
| run: docker compose -f demo/docker-compose.yml up -d --wait | |
| - name: Wait for the first scrape | |
| # Pas de sleep devine, ici non plus. `up --wait` a deja bloque sur les | |
| # healthchecks ; il reste a laisser UN scrape complet atterrir, car la | |
| # decouverte matche des noms de metriques et Prometheus ingere tout | |
| # l'endpoint d'un coup. On attend donc la serie, pas une duree : sortie | |
| # des qu'elle est la, echec franc et date si elle ne vient jamais. | |
| run: | | |
| MAX=90 # une seule source pour le budget, boucle et message compris | |
| for i in $(seq 1 "$MAX"); do | |
| if curl -sf http://localhost:9090/api/v1/label/__name__/values \ | |
| | grep -q '"gen_ai_'; then | |
| echo "series gen_ai visibles apres ${i}s"; exit 0 | |
| fi | |
| sleep 1 | |
| done | |
| echo "::error::aucune serie gen_ai apres ${MAX}s (emitter ou scrape KO)" | |
| exit 1 | |
| - name: Discover, forge and deploy for real | |
| env: {GRAFANA_URL: "http://localhost:3000", GRAFANA_USER: admin, GRAFANA_PASSWORD: admin} | |
| run: | | |
| python scripts/discover.py --out cap.json | |
| python - <<'PY' | |
| import json | |
| cap = json.load(open("cap.json")) | |
| print("dialects:", sorted({d for s in cap["signals"].values() for d in s})) | |
| PY | |
| python scripts/forge_dashboards.py --capability cap.json \ | |
| --blueprints auto --deploy --with-alerts --export-portable | |
| - name: Assert dashboards are live and queryable | |
| env: {GRAFANA_URL: "http://localhost:3000"} | |
| run: | | |
| python - <<'PY' | |
| import base64, json, urllib.request | |
| auth = base64.b64encode(b"admin:admin").decode() | |
| def get(p): | |
| r = urllib.request.Request( | |
| "http://localhost:3000" + p, | |
| headers={"Authorization": "Basic " + auth}) | |
| return json.load(urllib.request.urlopen(r)) | |
| ds = get("/api/search?tag=llmops-forge") | |
| assert len(ds) >= 5, f"expected >=5 dashboards, got {len(ds)}" | |
| for d in ds: | |
| full = get("/api/dashboards/uid/"+d["uid"]) | |
| assert full["dashboard"]["panels"], d["title"] | |
| print(f"{len(ds)} dashboards live:", [d["title"] for d in ds]) | |
| PY | |
| - name: Diagnose on failure (compose ps + container logs) | |
| if: failure() | |
| run: | | |
| docker compose -f demo/docker-compose.yml ps -a | |
| docker compose -f demo/docker-compose.yml logs --tail=150 | |
| - name: Teardown | |
| if: always() | |
| run: docker compose -f demo/docker-compose.yml down -v |