Merge pull request #8772 from Couchers-org/backend/feature/feature-flag-disk-cache

Backend: persist feature flags to a disk cache

Author: aapeliv

app/backend.dev.env

@@ -89,6 +89,7 @@ EXPERIMENTATION_ENABLED=0
 EXPERIMENTATION_PASS_ALL_GATES=1
 GROWTHBOOK_API_HOST=https://gbapi.couchershq.org
 GROWTHBOOK_CLIENT_KEY=sdk-cGmljeXu1BCzHffE
+GROWTHBOOK_CACHE_PATH=growthbook-features-cache.json
 
 # Moderation auto-approval deadline in seconds (0 to disable)
 MODERATION_AUTO_APPROVE_DEADLINE_SECONDS=10

app/backend/.gitignore

@@ -19,3 +19,6 @@ htmlcov/
 
 # test artifacts
 test_artifacts/
+
+# GrowthBook feature-flag disk cache (GROWTHBOOK_CACHE_PATH)
+growthbook-features-cache.json

app/backend/src/couchers/config.py

@@ -122,6 +122,9 @@
     # GrowthBook SDK configuration
     ("GROWTHBOOK_API_HOST", str, "https://cdn.growthbook.io"),
     ("GROWTHBOOK_CLIENT_KEY", str, ""),
+    # Disk path for the last-known-good feature payload, used as a cold-start fallback when GrowthBook
+    # is unreachable. Required when experimentation is enabled so we never start on in-code defaults.
+    ("GROWTHBOOK_CACHE_PATH", str, ""),
     # Moderation auto-approval deadline in seconds (0 to disable auto-approval)
     ("MODERATION_AUTO_APPROVE_DEADLINE_SECONDS", int),
     # User ID of the bot user for automated moderation actions
@@ -173,6 +176,8 @@ def check_config(cfg: dict[str, Any]) -> None:
     if cfg["EXPERIMENTATION_ENABLED"]:
         if not cfg["GROWTHBOOK_CLIENT_KEY"]:
             raise Exception("No GrowthBook client key but experimentation enabled")
+        if not cfg["GROWTHBOOK_CACHE_PATH"]:
+            raise Exception("No GrowthBook cache path but experimentation enabled")
 
 
 def make_config() -> dict[str, Any]:

app/backend/src/couchers/experimentation.py

@@ -19,7 +19,10 @@
 import json
 import logging
 import threading
+import time
 from collections.abc import Callable
+from pathlib import Path
+from tempfile import NamedTemporaryFile
 from typing import Any
 
 import urllib3
@@ -42,12 +45,19 @@
 _state_lock = threading.Lock()
 _refresh_stop = threading.Event()
 _refresh_thread: threading.Thread | None = None
+# Unix time of the last successful pull from GrowthBook (None until the first success). Set when we
+# load from the API or seed from the disk cache; drives the staleness metric.
+_last_fetch_time: float | None = None
 
 
 class ExperimentationNotInitializedError(Exception):
     """Raised when experimentation functions are called before initialization."""
 
 
+class GrowthBookUnavailableError(Exception):
+    """Raised at startup when features can't be fetched and there's no usable disk cache to fall back on."""
+
+
 def _fetch_features() -> dict[str, Any] | None:
     """Fetch the GrowthBook feature payload over HTTP. Returns None on failure."""
     api_host = config["GROWTHBOOK_API_HOST"].rstrip("/")
@@ -74,13 +84,48 @@ def _apply_response(response: dict[str, Any]) -> None:
         _state["savedGroups"] = response.get("savedGroups", {})
 
 
+def _set_last_fetch_time(when: float) -> None:
+    global _last_fetch_time
+    _last_fetch_time = when
+
+
+def seconds_since_last_fetch() -> float | None:
+    """Seconds since the last successful pull, or None if never pulled. Drives the staleness metric."""
+    when = _last_fetch_time
+    if when is None:
+        return None
+    return max(0.0, time.time() - when)
+
+
+def _write_cache(response: dict[str, Any]) -> None:
+    path = Path(config["GROWTHBOOK_CACHE_PATH"])
+    data = json.dumps({"fetched_at": time.time(), "response": response})
+    # Temp file alongside the target then rename: rename is atomic within a filesystem, so a reader
+    # never sees a half-written cache.
+    with NamedTemporaryFile("w", dir=path.parent, prefix=".growthbook-cache-", suffix=".tmp", delete=False) as f:
+        f.write(data)
+        tmp = Path(f.name)
+    tmp.replace(path)
+
+
+def _read_cache() -> tuple[dict[str, Any], float] | None:
+    """(response, fetched_at), or None if no cache file exists yet. A corrupt file raises."""
+    path = Path(config["GROWTHBOOK_CACHE_PATH"])
+    if not path.exists():
+        return None
+    payload = json.loads(path.read_text())
+    return payload["response"], payload["fetched_at"]
+
+
 def _refresh_loop() -> None:
     while not _refresh_stop.wait(_REFRESH_INTERVAL_SECONDS):
         response = _fetch_features()
         if response is not None:
             _apply_response(response)
+            _write_cache(response)
+            _set_last_fetch_time(time.time())
             logger.debug("GrowthBook features refreshed")
-        # On failure, keep last-known-good state and try again next tick.
+        # On a failed fetch, keep last-known-good state and retry next tick; the staleness metric climbs.
 
 
 def setup_experimentation() -> None:
@@ -107,6 +152,24 @@ def setup_experimentation() -> None:
     response = _fetch_features()
     if response is not None:
         _apply_response(response)
+        _write_cache(response)
+        _set_last_fetch_time(time.time())
+        logger.info("GrowthBook features loaded from API")
+    else:
+        # Unreachable at startup: fall back to the disk cache rather than booting on in-code defaults.
+        cached = _read_cache()
+        if cached is None:
+            raise GrowthBookUnavailableError(
+                "Could not fetch features from GrowthBook and no disk cache is available - refusing to "
+                "start on in-code feature-flag defaults"
+            )
+        cached_response, fetched_at = cached
+        _apply_response(cached_response)
+        _set_last_fetch_time(fetched_at)
+        logger.warning(
+            "GrowthBook unavailable at startup; loaded features from disk cache (%.0fs old)",
+            max(0.0, time.time() - fetched_at),
+        )
 
     with _state_lock:
         smoke_gb = GrowthBook(features=_state["features"], savedGroups=_state["savedGroups"])

app/backend/src/couchers/metrics.py

@@ -19,6 +19,7 @@
 from sqlalchemy.sql import distinct, func
 from sqlalchemy.sql.selectable import Select
 
+from couchers import experimentation
 from couchers.db import session_scope
 from couchers.helpers.completed_profile import has_completed_profile_expression
 from couchers.materialized_views import ClusterSubscriptionCount
@@ -633,6 +634,20 @@ def observe_moderation_queue_resolution_time(
 )
 
 
+# Recomputed at scrape time via the hacky-gauge mechanism, so it reflects live age. 0 when disabled
+# or never pulled.
+def _feature_flags_staleness_seconds() -> float:
+    return experimentation.seconds_since_last_fetch() or 0.0
+
+
+feature_flags_staleness_gauge: Gauge = Gauge(
+    "couchers_feature_flags_staleness_seconds",
+    "Seconds since feature flags were last successfully fetched from GrowthBook",
+    multiprocess_mode="mostrecent",
+)
+_set_hacky_gauges_funcs.append((feature_flags_staleness_gauge, _feature_flags_staleness_seconds))
+
+
 def create_prometheus_server(port: int) -> Any:
     """custom start method to fix problem descrbied in https://github.com/prometheus/client_python/issues/155"""
 

app/backend/src/tests/conftest.py

@@ -235,6 +235,7 @@ def testconfig():
     config["EXPERIMENTATION_PASS_ALL_GATES"] = True
     config["GROWTHBOOK_API_HOST"] = "https://cdn.growthbook.io"
     config["GROWTHBOOK_CLIENT_KEY"] = ""
+    config["GROWTHBOOK_CACHE_PATH"] = ""
 
     # Moderation auto-approval deadline - 0 disables, set in tests that need it
     config["MODERATION_AUTO_APPROVE_DEADLINE_SECONDS"] = 0

app/backend/src/tests/test_experimentation.py

@@ -1,11 +1,14 @@
+import json
+
+import pytest
 from growthbook.common_types import FeatureResult
 from sqlalchemy import select
 
 from couchers import experimentation
 from couchers.config import config
 from couchers.context import make_background_user_context, make_logged_out_context
 from couchers.db import session_scope
-from couchers.experimentation import _record_feature_usage
+from couchers.experimentation import GrowthBookUnavailableError, _record_feature_usage, setup_experimentation
 from couchers.i18n import LocalizationContext
 from couchers.models.logging import ExperimentExposure, FeatureUsage
 from couchers.proto import bugs_pb2
@@ -145,3 +148,68 @@ def test_global_evaluation_gets_global_force_on_flag(feature_flags):
 
 def test_global_evaluation_unknown_feature_returns_in_code_default(feature_flags):
     assert experimentation.get_global_string_value("does_not_exist", "my_default") == "my_default"
+
+
+@pytest.fixture
+def setup_isolation(monkeypatch, tmp_path):
+    """Run setup_experimentation() against a clean module state and a tmp cache path, and make sure the
+    background refresh thread it starts is stopped afterwards."""
+    monkeypatch.setattr(experimentation, "_initialized", False)
+    monkeypatch.setattr(experimentation, "_last_fetch_time", None)
+    monkeypatch.setattr(experimentation, "_state", {"features": {}, "savedGroups": {}})
+    monkeypatch.setitem(config, "EXPERIMENTATION_ENABLED", True)
+    monkeypatch.setitem(config, "GROWTHBOOK_CACHE_PATH", str(tmp_path / "cache.json"))
+    yield tmp_path / "cache.json"
+    experimentation._refresh_stop.set()
+    if experimentation._refresh_thread is not None:
+        experimentation._refresh_thread.join(timeout=5)
+    experimentation._refresh_stop.clear()
+    experimentation._refresh_thread = None
+
+
+def test_setup_writes_cache_and_records_fetch_time(setup_isolation, monkeypatch):
+    cache = setup_isolation
+    payload = {"features": {"f": {"defaultValue": True}}, "savedGroups": {}}
+    monkeypatch.setattr(experimentation, "_fetch_features", lambda: payload)
+
+    setup_experimentation()
+
+    assert experimentation._state["features"] == {"f": {"defaultValue": True}}
+    assert experimentation.seconds_since_last_fetch() is not None
+    written = json.loads(cache.read_text())
+    assert written["response"] == payload
+    assert "fetched_at" in written
+
+
+def test_setup_falls_back_to_disk_cache_when_fetch_fails(setup_isolation, monkeypatch):
+    cache = setup_isolation
+    cached_payload = {"features": {"cached": {"defaultValue": "x"}}, "savedGroups": {}}
+    cache.write_text(json.dumps({"fetched_at": 1000.0, "response": cached_payload}))
+    monkeypatch.setattr(experimentation, "_fetch_features", lambda: None)
+
+    setup_experimentation()
+
+    assert experimentation._state["features"] == {"cached": {"defaultValue": "x"}}
+    # fetch time reflects the cached pull time, so staleness is large immediately
+    staleness = experimentation.seconds_since_last_fetch()
+    assert staleness is not None and staleness > 0
+
+
+def test_setup_raises_when_fetch_fails_and_no_cache(setup_isolation, monkeypatch):
+    monkeypatch.setattr(experimentation, "_fetch_features", lambda: None)
+
+    with pytest.raises(GrowthBookUnavailableError):
+        setup_experimentation()
+
+
+def test_setup_raises_on_corrupt_cache(setup_isolation, monkeypatch):
+    cache = setup_isolation
+    cache.write_text("this is not json")
+    monkeypatch.setattr(experimentation, "_fetch_features", lambda: None)
+
+    with pytest.raises(json.JSONDecodeError):
+        setup_experimentation()
+
+
+def test_seconds_since_last_fetch_none_when_never_fetched(setup_isolation):
+    assert experimentation.seconds_since_last_fetch() is None

app/docker-compose.prod.yml

@@ -29,6 +29,8 @@ services:
     volumes:
       # auxiliary resources
       - "./data/aux/:/app/aux/:ro"
+      # persistent GrowthBook feature-flag cache (GROWTHBOOK_CACHE_PATH), survives deploys
+      - "./data/cache/:/data/cache/"
     expose:
       - 1751
       - 1753