wire dashboard to per-track metrics (#533)

.gitignore

@@ -43,3 +43,7 @@ docker/grafana/provisioning/dashboards/archive/
 
 # Version stamp for source trees with no .git (cmake/MoqxVersion.cmake).
 /VERSION
+
+# Python bytecode from local tooling.
+__pycache__/
+*.pyc

docker/config.docker.yaml

@@ -59,5 +59,6 @@ services:
 
 admin:
   port: ${MOQX_ADMIN_PORT}
+  track_metrics_enabled: true
   address: "${MOQX_BIND_ADDR}"
   plaintext: true

docker/docker-compose.yml

@@ -202,6 +202,7 @@ services:
     mem_limit: ${PROMETHEUS_MEM:-2g}
     volumes:
       - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - prometheus-targets:/etc/prometheus/targets:ro
       # TSDB backing store. Defaults to the named volume (persists across
       # redeploys — `compose down` without -v keeps it). Set PROMETHEUS_DATA_DIR
       # to an absolute host path to pin it on a specific/larger disk instead.
@@ -315,6 +316,29 @@ services:
         max-size: "10m"
         max-file: "3"
 
+  # Writes one Prometheus file_sd target per live namespace, read from the
+  # relay's namespace tree. /metrics/track takes a single namespace per
+  # request, and namespaces come and go with events, so the list cannot be
+  # static.
+  ns-targets:
+    container_name: moqx-ns-targets
+    image: python:3.12-alpine
+    restart: unless-stopped
+    profiles: [stats]
+    command: ["python3", "/app/namespace-targets.py"]
+    environment:
+      MOQX_STATE_URL: http://moqx:${MOQX_ADMIN_PORT:-8000}/state
+      MOQX_TARGETS_PATH: /targets/namespaces.json
+      MOQX_TARGETS_INTERVAL: "30"
+    volumes:
+      - ./prometheus/namespace-targets.py:/app/namespace-targets.py:ro
+      - prometheus-targets:/targets
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "3"
+
   # Turns the relay admin /state JSON into per-track Prometheus series (the
   # aggregate /metrics has no per-track labels). Prometheus scrapes it with a
   # target= param pointing at the relay /state (see the moqx-state job).
@@ -334,6 +358,7 @@ services:
         max-file: "3"
 
 volumes:
+  prometheus-targets:
   moqx-coredumps:
   prometheus-data:
   grafana-data:

docker/json-exporter/json-exporter.yml

@@ -1,31 +1,15 @@
 # json_exporter config for the moqx stats stack.
 #
-# Turns the relay admin /state JSON (which the aggregate /metrics does NOT carry)
-# into per-track Prometheus series so Grafana can graph subscribers per track
-# over time. Prometheus scrapes this exporter with ?module=moqx&target=<state-url>
-# (see the moqx-state job in prometheus.yml).
+# Turns admin JSON the aggregate /metrics does not carry into Prometheus
+# series. Prometheus scrapes this exporter with ?module=<name>&target=<url>
+# (see the moqx-state and moqx-info jobs in prometheus.yml).
 #
-# NOTE: per-track series scale with the number of distinct track names, so this
-# is higher-cardinality than the relay's own metrics — scraped at 15s to keep
-# the sample rate down.
+# Per-track series come from the relay's own /metrics/track, not from here:
+# both emitted moqx_track_subscribers, and /state's entries are transient and
+# undercount under load (openmoq/moqx#501).
 modules:
   moqx:
     metrics:
-      # One series per active subscription entry, keyed by track name.
-      - name: moqx_track
-        type: object
-        help: Per-track subscription stats from the relay admin /state
-        path: '{.services.default.subscriptions[*]}'
-        labels:
-          track: '{.track_name}'
-          is_publish: '{.is_publish}'
-          # namespace is a JSON array (e.g. ["com.wowza","moq","moqtest_all"]);
-          # slash-join it into a browsable path label (separator between
-          # elements only — no trailing "/").
-          namespace: '{.namespace[0]}{range .namespace[1:]}/{@}{end}'
-        values:
-          subscribers: '{.subscribers}'
-          forwarding_subscribers: '{.forwarding_subscribers}'
       # Top-level scalars for cross-checking against the aggregate /metrics.
       - name: moqx_state_active_sessions
         help: Active sessions reported by the admin /state

docker/prometheus/README.md

@@ -0,0 +1,44 @@
+# Prometheus scrape config
+
+## Per-track metrics (`moqx-track`)
+
+`/metrics/track` reports counters per live track. It takes one namespace per
+request, so each namespace is scraped separately.
+
+Scoping per namespace is not just tidiness. `limit` is a guard rather than a
+selector: a request matching more tracks than the limit returns 400 instead of
+truncating, because an arbitrary subset would give Prometheus a series set that
+reshuffles between scrapes. An unscoped scrape therefore fails as soon as the
+relay's *total* track count passes the limit, and takes every namespace with
+it, while per-namespace scrapes keep working until a *single* namespace passes
+it. Ceiling is `admin.track_metrics_endpoint_max_limit` (1000).
+
+Counting only happens when `admin.track_metrics_enabled` is true (the default);
+with it false the endpoint returns 503 rather than an empty scrape that would
+read as "no live tracks".
+
+## Target generation (`ns-targets`)
+
+Namespaces come and go with events, so the target list cannot be static.
+`namespace-targets.py` walks the relay's `/state` namespace tree every 30s and
+writes one target per namespace to a file Prometheus rereads without a restart.
+It emits only top-level namespaces. The endpoint matches a prefix, so scraping
+a parent and its child would return the same tracks twice and every aggregate
+would double-count; top-level namespaces do not overlap, and their prefixes
+still cover every track beneath them. If one grows past the limit it has to be
+split by descending a level.
+
+Targets are namespace values in the moq-transport safe form — `[A-Za-z0-9_]`
+passes through, every other byte becomes `.<hex>`, tuple elements join with
+`-`:
+
+    moq-test/interop           ->  moq.2dtest-interop
+    conf.example.com / room 1  ->  conf.2eexample.2ecom-room.201
+
+Relabelling turns each target into the `namespace` query parameter and points
+the scrape at the relay. The unencoded namespace is kept as `moqx_namespace`
+for display, since the encoded form is what lands in the metric labels.
+
+Series exist only while a track is live: they disappear when it ends and
+restart from zero if it returns, so counters need `rate()`/`increase()` rather
+than differences taken across a track's lifetime.

docker/prometheus/namespace-targets.py

@@ -0,0 +1,100 @@
+#!/usr/bin/env python3
+"""Generate Prometheus file_sd targets for per-track metric scrapes.
+
+/metrics/track takes one namespace per request and rejects a match wider than
+its limit, so each namespace is scraped separately. This walks the relay's
+namespace tree and writes one target per namespace; Prometheus rereads the file
+without a restart.
+
+Namespaces are written in the moq-transport safe form the endpoint expects:
+[A-Za-z0-9_] passes through, every other byte becomes .<hex>, and tuple
+elements are joined with '-'.
+"""
+import json
+import os
+import sys
+import tempfile
+import time
+import urllib.request
+
+STATE_URL = os.environ.get("MOQX_STATE_URL", "http://moqx:8000/state")
+OUT_PATH = os.environ.get("MOQX_TARGETS_PATH", "/targets/namespaces.json")
+INTERVAL = float(os.environ.get("MOQX_TARGETS_INTERVAL", "30"))
+
+_PASS = set(
+    "abcdefghijklmnopqrstuvwxyz" "ABCDEFGHIJKLMNOPQRSTUVWXYZ" "0123456789" "_"
+)
+
+
+def safe_element(element):
+    out = []
+    for byte in element.encode():
+        ch = chr(byte)
+        out.append(ch if ch in _PASS else ".%02x" % byte)
+    return "".join(out)
+
+
+def safe_namespace(tuple_elements):
+    return "-".join(safe_element(e) for e in tuple_elements)
+
+
+def top_level(tree):
+    """Only the tree's immediate children.
+
+    The endpoint matches a namespace prefix, so scraping both a parent and its
+    child returns the same tracks twice and every aggregate double-counts.
+    Top-level namespaces do not overlap each other and their prefixes still
+    cover every track beneath them.
+    """
+    out = []
+    for child in (tree.get("children") or {}).values():
+        full = child.get("full_namespace") or []
+        if full:
+            out.append(full)
+    return out
+
+
+def namespaces():
+    with urllib.request.urlopen(STATE_URL, timeout=10) as resp:
+        state = json.load(resp)
+    found = []
+    for service in (state.get("services") or {}).values():
+        tree = service.get("namespace_tree")
+        if tree:
+            found.extend(top_level(tree))
+    # A namespace can appear under more than one service.
+    return sorted({tuple(ns) for ns in found})
+
+
+def write(path, entries):
+    payload = [
+        {"targets": [safe_namespace(ns)], "labels": {"moqx_namespace": "/".join(ns)}}
+        for ns in entries
+    ]
+    body = json.dumps(payload, indent=2) + "\n"
+    if os.path.exists(path) and open(path).read() == body:
+        return False
+    # Rename into place so Prometheus never reads a partial file.
+    directory = os.path.dirname(path) or "."
+    fd, tmp = tempfile.mkstemp(dir=directory)
+    with os.fdopen(fd, "w") as handle:
+        handle.write(body)
+    # mkstemp makes it 0600 and this runs as root; Prometheus runs as nobody.
+    os.chmod(tmp, 0o644)
+    os.replace(tmp, path)
+    return True
+
+
+def main():
+    while True:
+        try:
+            entries = namespaces()
+            if write(OUT_PATH, entries):
+                print("wrote %d namespace target(s)" % len(entries), flush=True)
+        except Exception as exc:  # keep polling: the relay restarts
+            print("namespace target refresh failed: %s" % exc, file=sys.stderr, flush=True)
+        time.sleep(INTERVAL)
+
+
+if __name__ == "__main__":
+    main()

docker/prometheus/prometheus.yml

@@ -72,3 +72,25 @@ scrape_configs:
         target_label: __param_target
       - target_label: __address__
         replacement: json-exporter:7979
+
+  # Per-track counters from the relay (openmoq/moqx#533). One namespace per
+  # request: an unscoped scrape covers everything only until the total exceeds
+  # the limit, at which point it 400s and takes every namespace down with it,
+  # while per-namespace scrapes keep working until a single namespace exceeds
+  # it. Targets are namespaces in the safe form, generated from the relay's
+  # namespace tree by ns-targets and relabelled into the query.
+  - job_name: moqx-track
+    metrics_path: /metrics/track
+    scrape_interval: 15s
+    params:
+      limit: ['1000']
+    file_sd_configs:
+      - files: ['/etc/prometheus/targets/namespaces.json']
+        refresh_interval: 30s
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_namespace
+      - target_label: __address__
+        replacement: moqx:8000
+      - target_label: instance
+        replacement: moqx-relay